{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic 回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>dis2MHD</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>Wday</th>\n",
       "      <th>Yday</th>\n",
       "      <th>top_10_manager</th>\n",
       "      <th>...</th>\n",
       "      <th>top_5_manager</th>\n",
       "      <th>top_50_manager</th>\n",
       "      <th>top_1_manager</th>\n",
       "      <th>top_2_manager</th>\n",
       "      <th>top_15_manager</th>\n",
       "      <th>top_20_manager</th>\n",
       "      <th>top_30_manager</th>\n",
       "      <th>photos_count</th>\n",
       "      <th>features_count</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>5725.808989</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>176</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>10406.936420</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>164</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>3602.496322</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>108</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>6306.161401</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>109</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>13983.395510</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price       dis2MHD  Year  Month  Day  Wday  Yday  \\\n",
       "0        1.5         3   3000   5725.808989  2016      6   24     4   176   \n",
       "1        1.0         2   5465  10406.936420  2016      6   12     6   164   \n",
       "2        1.0         1   2850   3602.496322  2016      4   17     6   108   \n",
       "3        1.0         1   3275   6306.161401  2016      4   18     0   109   \n",
       "4        1.0         4   3350  13983.395510  2016      4   28     3   119   \n",
       "\n",
       "   top_10_manager       ...        top_5_manager  top_50_manager  \\\n",
       "0               1       ...                    1               1   \n",
       "1               1       ...                    1               1   \n",
       "2               1       ...                    1               1   \n",
       "3               1       ...                    1               1   \n",
       "4               0       ...                    0               1   \n",
       "\n",
       "   top_1_manager  top_2_manager  top_15_manager  top_20_manager  \\\n",
       "0              0              0               1               1   \n",
       "1              0              0               1               1   \n",
       "2              0              1               1               1   \n",
       "3              1              1               1               1   \n",
       "4              0              0               0               0   \n",
       "\n",
       "   top_30_manager  photos_count  features_count  interest_level  \n",
       "0               1             5               0               1  \n",
       "1               1            11               5               2  \n",
       "2               1             8               4               0  \n",
       "3               1             3               2               2  \n",
       "4               1             3               1               2  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = pd.read_csv(\"RentListingInquries_FE_train.csv\")\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>dis2MHD</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>Wday</th>\n",
       "      <th>Yday</th>\n",
       "      <th>top_10_manager</th>\n",
       "      <th>...</th>\n",
       "      <th>top_5_manager</th>\n",
       "      <th>top_50_manager</th>\n",
       "      <th>top_1_manager</th>\n",
       "      <th>top_2_manager</th>\n",
       "      <th>top_15_manager</th>\n",
       "      <th>top_20_manager</th>\n",
       "      <th>top_30_manager</th>\n",
       "      <th>photos_count</th>\n",
       "      <th>features_count</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.00000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.0</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "      <td>49278.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.210875</td>\n",
       "      <td>1.54083</td>\n",
       "      <td>3657.135882</td>\n",
       "      <td>6612.808665</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>5.014976</td>\n",
       "      <td>15.209850</td>\n",
       "      <td>2.922318</td>\n",
       "      <td>137.006575</td>\n",
       "      <td>0.606214</td>\n",
       "      <td>...</td>\n",
       "      <td>0.460043</td>\n",
       "      <td>0.947603</td>\n",
       "      <td>0.221235</td>\n",
       "      <td>0.302041</td>\n",
       "      <td>0.705508</td>\n",
       "      <td>0.771176</td>\n",
       "      <td>0.854600</td>\n",
       "      <td>5.603413</td>\n",
       "      <td>5.428893</td>\n",
       "      <td>1.616583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.495222</td>\n",
       "      <td>1.11295</td>\n",
       "      <td>2258.835488</td>\n",
       "      <td>3947.320372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.824431</td>\n",
       "      <td>8.280243</td>\n",
       "      <td>1.784835</td>\n",
       "      <td>25.868199</td>\n",
       "      <td>0.488593</td>\n",
       "      <td>...</td>\n",
       "      <td>0.498406</td>\n",
       "      <td>0.222828</td>\n",
       "      <td>0.415082</td>\n",
       "      <td>0.459148</td>\n",
       "      <td>0.455819</td>\n",
       "      <td>0.420080</td>\n",
       "      <td>0.352507</td>\n",
       "      <td>3.626500</td>\n",
       "      <td>3.922919</td>\n",
       "      <td>0.626215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>4.447797</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>92.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2500.000000</td>\n",
       "      <td>3707.371517</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>3150.000000</td>\n",
       "      <td>6153.217534</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>137.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.00000</td>\n",
       "      <td>4100.000000</td>\n",
       "      <td>8978.502986</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>160.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.500000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>58020.000000</td>\n",
       "      <td>44065.820784</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>181.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          bathrooms     bedrooms         price       dis2MHD     Year  \\\n",
       "count  49278.000000  49278.00000  49278.000000  49278.000000  49278.0   \n",
       "mean       1.210875      1.54083   3657.135882   6612.808665   2016.0   \n",
       "std        0.495222      1.11295   2258.835488   3947.320372      0.0   \n",
       "min        0.000000      0.00000     43.000000      4.447797   2016.0   \n",
       "25%        1.000000      1.00000   2500.000000   3707.371517   2016.0   \n",
       "50%        1.000000      1.00000   3150.000000   6153.217534   2016.0   \n",
       "75%        1.000000      2.00000   4100.000000   8978.502986   2016.0   \n",
       "max        5.500000      6.00000  58020.000000  44065.820784   2016.0   \n",
       "\n",
       "              Month           Day          Wday          Yday  top_10_manager  \\\n",
       "count  49278.000000  49278.000000  49278.000000  49278.000000    49278.000000   \n",
       "mean       5.014976     15.209850      2.922318    137.006575        0.606214   \n",
       "std        0.824431      8.280243      1.784835     25.868199        0.488593   \n",
       "min        4.000000      1.000000      0.000000     92.000000        0.000000   \n",
       "25%        4.000000      8.000000      1.000000    114.000000        0.000000   \n",
       "50%        5.000000     15.000000      3.000000    137.000000        1.000000   \n",
       "75%        6.000000     22.000000      4.000000    160.000000        1.000000   \n",
       "max        6.000000     31.000000      6.000000    181.000000        1.000000   \n",
       "\n",
       "            ...        top_5_manager  top_50_manager  top_1_manager  \\\n",
       "count       ...         49278.000000    49278.000000   49278.000000   \n",
       "mean        ...             0.460043        0.947603       0.221235   \n",
       "std         ...             0.498406        0.222828       0.415082   \n",
       "min         ...             0.000000        0.000000       0.000000   \n",
       "25%         ...             0.000000        1.000000       0.000000   \n",
       "50%         ...             0.000000        1.000000       0.000000   \n",
       "75%         ...             1.000000        1.000000       0.000000   \n",
       "max         ...             1.000000        1.000000       1.000000   \n",
       "\n",
       "       top_2_manager  top_15_manager  top_20_manager  top_30_manager  \\\n",
       "count   49278.000000    49278.000000    49278.000000    49278.000000   \n",
       "mean        0.302041        0.705508        0.771176        0.854600   \n",
       "std         0.459148        0.455819        0.420080        0.352507   \n",
       "min         0.000000        0.000000        0.000000        0.000000   \n",
       "25%         0.000000        0.000000        1.000000        1.000000   \n",
       "50%         0.000000        1.000000        1.000000        1.000000   \n",
       "75%         1.000000        1.000000        1.000000        1.000000   \n",
       "max         1.000000        1.000000        1.000000        1.000000   \n",
       "\n",
       "       photos_count  features_count  interest_level  \n",
       "count  49278.000000    49278.000000    49278.000000  \n",
       "mean       5.603413        5.428893        1.616583  \n",
       "std        3.626500        3.922919        0.626215  \n",
       "min        0.000000        0.000000        0.000000  \n",
       "25%        4.000000        2.000000        1.000000  \n",
       "50%        5.000000        5.000000        2.000000  \n",
       "75%        7.000000        8.000000        2.000000  \n",
       "max       68.000000       39.000000        2.000000  \n",
       "\n",
       "[8 rows x 21 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 各属性的统计特性\n",
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAELCAYAAAARNxsIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGyVJREFUeJzt3X2wHXWd5/H3J+FBfEyQC5VJwgQ1\nM4o6Rr1ClClH0YGAOwYpUChHIkNtxAKF0rUEd5aMPMyM4yArM8pupogE1yFERIkajBkGZEGeAoaH\nEFmuyMCVLAmGh6ArFOGzf/TvysnNyb2dS5+cnNzPq6rrdH/7132+h1vwpfv361/LNhEREU2Y0O0E\nIiJi15GiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIas1u3E9jR\n9tlnH8+YMaPbaURE9JTbb7/9Mdt9o7Ubd0VlxowZrFq1qttpRET0FEn/Uaddx25/SXqJpFsl3Slp\njaQvlvglkn4paXVZZpW4JF0oaUDSXZLe1nKueZLuL8u8lvjbJd1djrlQkjr1eyIiYnSdvFJ5BjjU\n9tOSdgdukHR12fc521cMa38EMLMsBwMXAQdL2htYAPQDBm6XtMz246XNfOBmYDkwB7iaiIjoio5d\nqbjydNncvSwjTYk8F7i0HHczMEnSFOBwYKXtjaWQrATmlH2vtH2Tq6mWLwWO6tTviYiI0XV09Jek\niZJWA+upCsMtZdd55RbXBZL2LLGpwMMthw+W2EjxwTbxiIjoko4WFdubbc8CpgEHSXoTcCbweuAd\nwN7A50vzdv0hHkN8K5LmS1oladWGDRu281dERERdO+Q5FdtPANcBc2yvK7e4ngG+ARxUmg0C01sO\nmwY8Mkp8Wpt4u+9faLvfdn9f36gj4iIiYow6OfqrT9Kksr4X8H7g56UvhDJS6yjgnnLIMuCEMgps\nNvCk7XXACuAwSZMlTQYOA1aUfZskzS7nOgG4qlO/JyIiRtfJ0V9TgMWSJlIVr6W2fyDp3yX1Ud2+\nWg2cXNovB44EBoDfAicC2N4o6RzgttLubNsby/ongUuAvahGfWXkV0REF2m8vaO+v7/fefgxImL7\nSLrddv9o7cbdE/URsfM75J8O6XYKu7wbP3VjR86bCSUjIqIxKSoREdGYFJWIiGhMikpERDQmRSUi\nIhqTohIREY1JUYmIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlR\niYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIa07GiIuklkm6VdKekNZK+WOIH\nSLpF0v2SLpe0R4nvWbYHyv4ZLec6s8Tvk3R4S3xOiQ1IOqNTvyUiIurp5JXKM8Chtt8CzALmSJoN\nfAm4wPZM4HHgpNL+JOBx268DLijtkHQgcBzwRmAO8HVJEyVNBL4GHAEcCBxf2kZERJd0rKi48nTZ\n3L0sBg4FrijxxcBRZX1u2absf58klfgS28/Y/iUwABxUlgHbD9h+FlhS2kZERJd0tE+lXFGsBtYD\nK4FfAE/Yfq40GQSmlvWpwMMAZf+TwKtb48OO2Va8XR7zJa2StGrDhg1N/LSIiGijo0XF9mbbs4Bp\nVFcWb2jXrHxqG/u2N94uj4W2+2339/X1jZ54RESMyQ4Z/WX7CeA6YDYwSdJuZdc04JGyPghMByj7\nXwVsbI0PO2Zb8YiI6JJOjv7qkzSprO8FvB9YC1wLHFOazQOuKuvLyjZl/7/bdokfV0aHHQDMBG4F\nbgNmltFke1B15i/r1O+JiIjR7TZ6kzGbAiwuo7QmAEtt/0DSvcASSecCPwMuLu0vBr4paYDqCuU4\nANtrJC0F7gWeA06xvRlA0qnACmAisMj2mg7+noiIGEXHiortu4C3tok/QNW/Mjz+O+DYbZzrPOC8\nNvHlwPIXnWxERDQiT9RHRERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmI\niMakqERERGNSVCIiojGjFhVJL5M0oaz/kaQPStq986lFRESvqXOlcj3wEklTgWuAE4FLOplURET0\npjpFRbZ/CxwN/JPtD1G9Ez4iImILtYqKpHcCHwV+WGKdnDI/IiJ6VJ2icjpwJvDd8m6T11C9aCsi\nImILo15x2P4J8BNJLyvbDwCf7nRiERHRe+qM/npneVvj2rL9Fklf73hmERHRc+rc/vrvwOHArwFs\n3wm8u5NJRUREb6r18KPth4eFNncgl4iI6HF1RnE9LOldgCXtQdWfsrazaUVERC+qc6VyMnAKMBUY\nBGaV7YiIiC2MWlRsP2b7o7b3s72v7b+0/evRjpM0XdK1ktZKWiPptBL/G0m/krS6LEe2HHOmpAFJ\n90k6vCU+p8QGJJ3REj9A0i2S7pd0ebmSioiILqkz+muxpEkt25MlLapx7ueAz9p+AzAbOEXS0JP4\nF9ieVZbl5bwHAscBbwTmAF+XNFHSROBrwBFUT/If33KeL5VzzQQeB06qkVdERHRIndtff2L7iaEN\n248Dbx3tINvrbN9R1jdR9cNMHeGQucAS28/Y/iUwABxUlgHbD9h+FlgCzJUk4FDginL8YuCoGr8n\nIiI6pE5RmSBp8tCGpL3ZzmlaJM2gKkS3lNCpku6StKjl3FOB1lFmgyW2rfirgSdsPzcsHhERXVKn\nqJwP/FTSOZLOAX4K/EPdL5D0cuA7wOm2nwIuAl5L1eG/rpwfQG0O9xji7XKYL2mVpFUbNmyom3pE\nRGynOh31lwLHAI8C64GjbX+zzsnLe1e+A3zL9pXlfI/a3mz7eeBfqG5vQXWlMb3l8GnAIyPEHwMm\nSdptWLzdb1hou992f19fX53UIyJiDOq++fHnwJXAVcDTkvYf7YDS53ExsNb2V1riU1qafQi4p6wv\nA46TtKekA4CZwK3AbcDMMtJrD6rO/GW2TTWx5THl+Hklv4iI6JJR+0YkfQpYQHWlspnqtpOBPxnl\n0EOAjwF3S1pdYl+gGr01q5zjQeATAGUG5KXAvVQjx06xvbnkcCqwApgILLK9ppzv88ASSecCP6Mq\nYhER0SV1OtxPA/64zrMprWzfQPt+j+UjHHMecF6b+PJ2x5UZkw8aHo+IiO6oc/vrYeDJTicSERG9\nr86VygPAdZJ+CDwzFGztJ4mIiIB6ReWhsuxRloiIiLbqvPnxiwCSXmb7N51PKSIielXe/BgREY3J\nmx8jIqIxefNjREQ0Jm9+jIiIxuTNjxER0ZgRr1TKC7I+ZvujOyifiIjoYSNeqZS5t+buoFwiIqLH\n1elTuVHSPwOXA79/TmXorY4RERFD6hSVd5XPs1tipnqVb0RExO+N1qcyAbjI9tIdlE9ERPSw0fpU\nngdO3UG5REREj6szpHilpP8iabqkvYeWjmcWERE9p06fyl+Vz9ZnUwy8pvl0IiKil9WZpfiAHZFI\nRET0vjrvqD+hXdz2pc2nExERvazO7a93tKy/BHgfcAeQohIREVuoc/vrU63bkl4FfLNjGUVERM+q\nNfX9ML8FZo7WqIwWu1bSWklrJJ1W4ntLWinp/vI5ucQl6UJJA5LukvS2lnPNK+3vlzSvJf52SXeX\nYy6UpDH8noiIaEidNz9+X9KysvwAuA+4qsa5nwM+a/sNwGzgFEkHAmcA19ieCVxTtgGOoCpWM4H5\nwEXl+/cGFgAHAwcBC4YKUWkzv+W4OTXyioiIDqnTp/KPLevPAf9he3C0g2yvA9aV9U2S1lJNnz8X\neE9pthi4Dvh8iV9q28DNkiZJmlLarrS9EUDSSmCOpOuAV9q+qcQvBY4Crq7xmyIiogPqFJWHgHW2\nfwcgaS9JM2w/WPdLJM0A3grcAuxXCg6210natzSbCrS+YXKwxEaKD7aJR0REl9TpU/k28HzL9uYS\nq0XSy4HvAKfbfmqkpm1iHkO8XQ7zJa2StGrDhg2jpRwREWNUp6jsZvvZoY2yvkedk0vanaqgfMv2\nlSX8aLmtRflcX+KDwPSWw6cBj4wSn9YmvhXbC2332+7v6+urk3pERIxBnaKyQdIHhzYkzQUeG+2g\nMhLrYmCt7a+07FoGDI3gmscLnf7LgBPKKLDZwJPlNtkK4DBJk0sH/WHAirJvk6TZ5btOoN4AgoiI\n6JA6fSonA98qL+qC6gqh7VP2wxwCfAy4W9LqEvsC8PfAUkknUfXXHFv2LQeOBAaohi2fCGB7o6Rz\ngNtKu7OHOu2BTwKXAHtRddCnkz4ioovqPPz4C2B26RuR7U11Tmz7Btr3e0D1VP7w9mbLSStb9y0C\nFrWJrwLeVCefiIjovDrPqfytpEm2ny5DgydLOndHJBcREb2lTp/KEbafGNqw/TjVbaqIiIgt1Ckq\nEyXtObQhaS9gzxHaR0TEOFWno/5/AddI+gbVcyB/RfUkfERExBbqdNT/g6S7gPeX0Dm2V3Q2rYiI\n6EV1rlQAfgbsTnWl8rPOpRMREb2szuivDwO3AscAHwZukXRMpxOLiIjeU+dK5b8C77C9HkBSH/Bv\nwBWdTCwiInpPndFfE4YKSvHrmsdFRMQ4U+dK5UeSVgCXle2PUE2pEhERsYU6o78+J+lo4E+ppl1Z\naPu7Hc8sIiJ6Tq3RX2Xa+itHbRgREeNa+kYiIqIxKSoREdGYbRYVSdeUzy/tuHQiIqKXjdSnMkXS\nnwEflLSEYe9GsX1HRzOLiIieM1JROQs4g+rd718Zts/AoZ1KKiIietM2i4rtK4ArJP032+fswJwi\nIqJH1XlO5RxJHwTeXULX2f5BZ9OKiIheVGdCyb8DTgPuLctpJRYREbGFOg8/fgCYZft5AEmLqaa/\nP7OTiUVERO+p+5zKpJb1V9U5QNIiSesl3dMS+xtJv5K0uixHtuw7U9KApPskHd4Sn1NiA5LOaIkf\nIOkWSfdLulzSHjV/S0REdEidovJ3wM8kXVKuUm4H/rbGcZcAc9rEL7A9qyzLASQdCBwHvLEc83VJ\nEyVNBL4GHAEcCBxf2gJ8qZxrJvA4cFKNnCIiooNGLSq2LwNmU839dSXwTttLahx3PbCxZh5zgSW2\nn7H9S2AAOKgsA7YfsP0ssASYK0lUQ5qH3umyGDiq5ndFRESH1Lr9ZXud7WW2r7L9f1/kd54q6a5y\ne2xyiU0FHm5pM1hi24q/GnjC9nPD4hER0UU7eu6vi4DXArOAdcD5Ja42bT2GeFuS5ktaJWnVhg0b\nti/jiIiobYcWFduP2t5cRpL9C9XtLaiuNKa3NJ0GPDJC/DFgkqTdhsW39b0Lbffb7u/r62vmx0RE\nxFZGLCqSJrSO3nqxJE1p2fwQMHTuZcBxkvaUdAAwE7gVuA2YWUZ67UHVmb/MtoFrgWPK8fOAq5rK\nMyIixmbE51RsPy/pTkn7235oe04s6TLgPcA+kgaBBcB7JM2iulX1IPCJ8j1rJC2lerjyOeAU25vL\neU4FVgATgUW215Sv+DywRNK5VM/NXLw9+UVERPPqPPw4BVgj6VbgN0NB2x8c6SDbx7cJb/M//LbP\nA85rE18OLG8Tf4AXbp9FRMROoE5R+WLHs4iIiF1CnQklfyLpD4GZtv9N0kupbkVFRERsoc6Ekv+Z\n6iHD/1lCU4HvdTKpiIjoTXWGFJ8CHAI8BWD7fmDfTiYVERG9qU5ReaZMkQJAeTZkmw8aRkTE+FWn\nqPxE0heAvST9OfBt4PudTSsiInpRnaJyBrABuJvquZLlwF93MqmIiOhNdUZ/PV+mvL+F6rbXfeWJ\n9oiIiC2MWlQkfQD4H8AvqCZyPEDSJ2xf3enkIiKit9R5+PF84L22BwAkvRb4IZCiEhERW6jTp7J+\nqKAUDwDrO5RPRET0sG1eqUg6uqyukbQcWErVp3Is1ezBERERWxjp9tdftKw/CvxZWd8ATN66eURE\njHfbLCq2T9yRiURERO+rM/rrAOBTwIzW9qNNfR8REeNPndFf36N6D8r3gec7m05ERPSyOkXld7Yv\n7HgmERHR8+oUla9KWgD8GHhmKGj7jo5lFRERPalOUXkz8DHgUF64/eWyHbFTeujsN3c7hXFh/7Pu\n7nYKsZOpU1Q+BLymdfr7iIiIduo8UX8nMKnTiURERO+rc6WyH/BzSbexZZ9KhhRHRMQW6hSVBWM5\nsaRFwH+imjvsTSW2N3A51TMvDwIftv24JAFfBY4Efgt8fGgggKR5vPD+lnNtLy7xtwOXAHtRvePl\ntEzJHxHRXaPe/rL9k3ZLjXNfAswZFjsDuMb2TOCasg1wBDCzLPOBi+D3RWgBcDBwELBA0tAUMReV\ntkPHDf+uiIjYwUYtKpI2SXqqLL+TtFnSU6MdZ/t6YOOw8FxgcVlfDBzVEr/UlZuBSZKmAIcDK21v\ntP04sBKYU/a90vZN5erk0pZzRUREl9R58+MrWrclHUV11TAW+9leV867TtK+JT4VeLil3WCJjRQf\nbBNvS9J8qqsa9t9//zGmHhERo6kz+msLtr9H88+oqN1XjSHelu2Ftvtt9/f19Y0xxYiIGE2dCSWP\nbtmcAPQzwn/AR/GopCnlKmUKL7zsaxCY3tJuGvBIib9nWPy6Ep/Wpn1ERHRRnSuVv2hZDgc2UfWB\njMUyYF5Znwdc1RI/QZXZwJPlNtkK4DBJk0sH/WHAirJvk6TZZeTYCS3nioiILqnTpzKm96pIuozq\nKmMfSYNUo7j+Hlgq6STgIaq3SEI1JPhIYIBqSPGJ5bs3SjqHF940ebbtoc7/T/LCkOKryxIREV00\n0uuEzxrhONs+Z6QT2z5+G7ve1+5kwCnbOM8iYFGb+CrgTSPlEBERO9ZIVyq/aRN7GXAS8GpgxKIS\nERHjz0ivEz5/aF3SK4DTqG5LLQHO39ZxERExfo3Yp1KeaP8M8FGqhxXfVh5CjIiI2MpIfSpfBo4G\nFgJvtv30DssqIiJ60khDij8L/AHVZI6PtEzVsqnONC0RETH+jNSnst1P20dExPiWwhEREY1JUYmI\niMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JU\nIiKiMSkqERHRmBSViIhoTIpKREQ0pitFRdKDku6WtFrSqhLbW9JKSfeXz8klLkkXShqQdJekt7Wc\nZ15pf7+ked34LRER8YJuXqm81/Ys2/1l+wzgGtszgWvKNsARwMyyzAcugqoIAQuAg4GDgAVDhSgi\nIrpjZ7r9NRdYXNYXA0e1xC915WZgkqQpwOHAStsbbT8OrATm7OikIyLiBd0qKgZ+LOl2SfNLbD/b\n6wDK574lPhV4uOXYwRLbVjwiIrpkm++o77BDbD8iaV9gpaSfj9BWbWIeIb71CarCNR9g//33395c\nIyKipq5cqdh+pHyuB75L1SfyaLmtRflcX5oPAtNbDp8GPDJCvN33LbTdb7u/r6+vyZ8SEREtdnhR\nkfQySa8YWgcOA+4BlgFDI7jmAVeV9WXACWUU2GzgyXJ7bAVwmKTJpYP+sBKLiIgu6cbtr/2A70oa\n+v5/tf0jSbcBSyWdBDwEHFvaLweOBAaA3wInAtjeKOkc4LbS7mzbG3fcz4iIiOF2eFGx/QDwljbx\nXwPvaxM3cMo2zrUIWNR0jhERMTY705DiiIjocSkqERHRmG4NKe4Jb//cpd1OYZd3+5dP6HYKEdGg\nXKlERERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmIiMakqERERGNSVCIi\nojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMT1fVCTNkXSf\npAFJZ3Q7n4iI8ayni4qkicDXgCOAA4HjJR3Y3awiIsavni4qwEHAgO0HbD8LLAHmdjmniIhxq9eL\nylTg4ZbtwRKLiIgu2K3bCbxIahPzVo2k+cD8svm0pPs6mlV37QM81u0k6tI/zut2CjuTnvrbAbCg\n3b+C41ZP/f306e3+2/1hnUa9XlQGgekt29OAR4Y3sr0QWLijkuomSats93c7j9h++dv1tvz9Kr1+\n++s2YKakAyTtARwHLOtyThER41ZPX6nYfk7SqcAKYCKwyPaaLqcVETFu9XRRAbC9HFje7Tx2IuPi\nNt8uKn+73pa/HyB7q37tiIiIMen1PpWIiNiJpKjsIjJdTe+StEjSekn3dDuX2D6Spku6VtJaSWsk\nndbtnLott792AWW6mv8D/DnVMOvbgONt39vVxKIWSe8GngYutf2mbucT9UmaAkyxfYekVwC3A0eN\n53/3cqWya8h0NT3M9vXAxm7nEdvP9jrbd5T1TcBaxvmsHikqu4ZMVxPRZZJmAG8FbuluJt2VorJr\nqDVdTUR0hqSXA98BTrf9VLfz6aYUlV1DrelqIqJ5knanKijfsn1lt/PpthSVXUOmq4noAkkCLgbW\n2v5Kt/PZGaSo7AJsPwcMTVezFlia6Wp6h6TLgJuAP5Y0KOmkbucUtR0CfAw4VNLqshzZ7aS6KUOK\nIyKiMblSiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlApD00xptTpf00g7nMWu0\n5xwkfVzSPzf8vY2fM8anFJUIwPa7ajQ7HdiuolJeS7A9ZgHj+uG56G0pKhGApKfL53skXSfpCkk/\nl/QtVT4N/AFwraRrS9vDJN0k6Q5J3y6TCiLpQUlnSboBOFbSayX9SNLtkv63pNeXdsdKukfSnZKu\nL1PsnA18pDyZ/ZEaefdJ+o6k28pyiKQJJYdJLe0GJO3Xrn3j/zBjXNut2wlE7ITeCryRalLOG4FD\nbF8o6TPAe20/Jmkf4K+B99v+jaTPA5+hKgoAv7P9pwCSrgFOtn2/pIOBrwOHAmcBh9v+laRJtp+V\ndBbQb/vUmrl+FbjA9g2S9gdW2H6DpKuADwHfKN/5oO1HJf3r8PbAG17kP6+I30tRidjarbYHASSt\nBmYANwxrMxs4ELixmlOQPajm7xpyeTn+5cC7gG+XdgB7ls8bgUskLQXGOrvt+4EDW879yvIGwsup\nitY3qCYYvXyU9hGNSFGJ2NozLeubaf/viYCVto/fxjl+Uz4nAE/YnjW8ge2Ty1XEB4DVkrZqU8ME\n4J22/98WyUk3Aa+T1AccBZw7SvsxfHXE1tKnElHfJmDo/+pvBg6R9DoASS+V9EfDDygvbPqlpGNL\nO0l6S1l/re1bbJ8FPEb1TpzW76jjx1QzVFPOOat8r4HvAl+hmpb91yO1j2hKikpEfQuBqyVda3sD\n8HHgMkl3URWZ12/juI8CJ0m6E1gDzC3xL0u6W9I9wPXAncC1VLenanXUA58G+iXdJele4OSWfZcD\nf8kLt75Gax/xomXq+4iIaEyuVCIiojHpqI/YSUk6EThtWPhG26d0I5+IOnL7KyIiGpPbXxER0ZgU\nlYiIaEyKSkRENCZFJSIiGpOiEhERjfn/hUdt8WfuoLkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa503630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target 分布，看看各类样本分布是否均衡\n",
    "sns.countplot(train_data.interest_level);\n",
    "pyplot.xlabel('interest_level');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡。交叉验证对分类任务缺省的是采用StratifiedKFold，在每折采样时根据各类样本按比例采样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置X，Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train_data['interest_level'] \n",
    "train_data = train_data.drop( \"interest_level\", axis=1)\n",
    "X_train = np.array(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "#X_test = ss_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression，默认"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [ 0.71656712  0.70065747  0.70195158  0.70000102  0.71189847]\n",
      "cv logloss is: 0.706215130764\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print 'logloss of each fold is: ',-loss\n",
    "print'cv logloss is:', -loss.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） 目标函数为：J = sum(logloss(f(xi), yi)) + C* penalty\n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同： 设置候选参数集合 调用GridSearchCV 调用fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda2\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  0.2848    ,   0.76240005,   0.61219993,   1.15599999,\n",
       "          1.02580004,   1.31160007,   4.05360007,   1.4335999 ,\n",
       "         63.97359991,   1.85380001,  74.09300008,   1.31920009,\n",
       "         85.03419995,   1.3678    ]),\n",
       " 'mean_score_time': array([ 0.00659995,  0.00619993,  0.00640001,  0.00680008,  0.00620003,\n",
       "         0.00620003,  0.00639997,  0.00680003,  0.00620003,  0.00739994,\n",
       "         0.004     ,  0.00499992,  0.004     ,  0.00460005]),\n",
       " 'mean_test_score': array([-0.77055372, -0.75533987, -0.71016491, -0.71217248, -0.70630672,\n",
       "        -0.70644558, -0.70620028, -0.70621544, -0.70623771, -0.70626796,\n",
       "        -0.70625445, -0.70633563, -0.7062882 , -0.70634862]),\n",
       " 'mean_train_score': array([-0.7704549 , -0.75070777, -0.70946257, -0.71104421, -0.70525713,\n",
       "        -0.70536664, -0.70511018, -0.70509699, -0.70507159, -0.70505055,\n",
       "        -0.70506234, -0.70504142, -0.70505554, -0.70504104]),\n",
       " 'param_C': masked_array(data = [0.001 0.001 0.01 0.01 0.1 0.1 1 1 10 10 100 100 1000 1000],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'param_penalty': masked_array(data = ['l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2'],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([14, 13, 11, 12,  7, 10,  1,  2,  3,  5,  4,  8,  6,  9]),\n",
       " 'split0_test_score': array([-0.77152617, -0.76476016, -0.71810618, -0.72064293, -0.7164366 ,\n",
       "        -0.7164729 , -0.71659639, -0.71656712, -0.7165894 , -0.71655601,\n",
       "        -0.71659612, -0.71654295, -0.71662449, -0.71654089]),\n",
       " 'split0_train_score': array([-0.76921152, -0.74914271, -0.70701914, -0.70871031, -0.70269178,\n",
       "        -0.7028054 , -0.70253204, -0.70252419, -0.70250359, -0.70249064,\n",
       "        -0.70249789, -0.70248617, -0.70250927, -0.70248595]),\n",
       " 'split1_test_score': array([-0.77018624, -0.75496417, -0.70530295, -0.7076485 , -0.70075953,\n",
       "        -0.70100692, -0.70058312, -0.70065747, -0.70067918, -0.70080464,\n",
       "        -0.70065552, -0.70090524, -0.70078507, -0.70092015]),\n",
       " 'split1_train_score': array([-0.77116782, -0.75167027, -0.71088701, -0.71236847, -0.70667145,\n",
       "        -0.70677348, -0.70653061, -0.70650773, -0.70647842, -0.70645295,\n",
       "        -0.70648487, -0.70644421, -0.70645402, -0.70644383]),\n",
       " 'split2_test_score': array([-0.76975654, -0.75202018, -0.70712479, -0.70885821, -0.70225694,\n",
       "        -0.70231913, -0.70195456, -0.70195158, -0.70193631, -0.70193804,\n",
       "        -0.70193894, -0.70195975, -0.70195628, -0.7019652 ]),\n",
       " 'split2_train_score': array([-0.77128607, -0.75126743, -0.71040462, -0.71201955, -0.7063267 ,\n",
       "        -0.70643265, -0.70618715, -0.70618983, -0.70617884, -0.70617498,\n",
       "        -0.70617748, -0.70617218, -0.70617606, -0.70617185]),\n",
       " 'split3_test_score': array([-0.77001719, -0.74858386, -0.70546747, -0.70760132, -0.70024801,\n",
       "        -0.70059896, -0.69998935, -0.70000102, -0.69995642, -0.69995508,\n",
       "        -0.69995084, -0.70001456, -0.69994526, -0.70002771]),\n",
       " 'split3_train_score': array([-0.77090292, -0.75151597, -0.71098581, -0.71238731, -0.70681755,\n",
       "        -0.70692432, -0.7066771 , -0.70666496, -0.70664734, -0.70661399,\n",
       "        -0.70663452, -0.70660187, -0.70661815, -0.7066016 ]),\n",
       " 'split4_test_score': array([-0.77128231, -0.75636911, -0.71482203, -0.7161102 , -0.71183101,\n",
       "        -0.71182852, -0.71187646, -0.71189847, -0.7120257 , -0.7120845 ,\n",
       "        -0.71212928, -0.71225414, -0.71212834, -0.71228766]),\n",
       " 'split4_train_score': array([-0.76970614, -0.74994245, -0.70801628, -0.70973544, -0.70377818,\n",
       "        -0.70389736, -0.70362402, -0.70359822, -0.70354974, -0.7035202 ,\n",
       "        -0.70351693, -0.70350268, -0.70352019, -0.70350194]),\n",
       " 'std_fit_time': array([  1.89568140e-02,   7.20349648e-02,   4.40926396e-02,\n",
       "          4.98477119e-02,   1.38002031e-01,   4.44683764e-02,\n",
       "          1.44942026e+00,   1.13741120e-01,   3.47476231e+01,\n",
       "          3.13990660e-01,   3.85570908e+01,   2.94516147e-01,\n",
       "          5.34184442e+01,   3.02117487e-01]),\n",
       " 'std_score_time': array([  7.99977784e-04,   7.48366451e-04,   1.35636768e-03,\n",
       "          1.32664003e-03,   3.99971008e-04,   1.83300148e-03,\n",
       "          1.19996866e-03,   4.00114074e-04,   1.72047893e-03,\n",
       "          1.74356995e-03,   9.53674316e-08,   8.94415679e-04,\n",
       "          6.32409699e-04,   4.89784582e-04]),\n",
       " 'std_test_score': array([ 0.00071203,  0.00541508,  0.00528595,  0.0052841 ,  0.00658822,\n",
       "         0.00648573,  0.00675964,  0.00674026,  0.0067753 ,  0.00675478,\n",
       "         0.00679973,  0.00675014,  0.00678586,  0.00674983]),\n",
       " 'std_train_score': array([ 0.00083743,  0.00099277,  0.00163088,  0.00152767,  0.00169396,\n",
       "         0.00168883,  0.00170225,  0.00170345,  0.00170873,  0.00170708,\n",
       "         0.00171485,  0.00170754,  0.00170258,  0.00170758])}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view the complete results (list of named tuples)\n",
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7062002844\n",
      "{'penalty': 'l1', 'C': 1}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果最佳值在候选参数的边缘，最好再尝试更大的候选参数或更小的候选参数，直到找到拐点。 l1, c=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd8VdeV6PHfUkENAUIIECAQzfRm\nRLEBm2ZDXHHD4BrbDMkbe1ImL4kTzySxk8k4L5Myk8mMCy4kprhiMDZggzHY2IAx3XTTQTR1oX7v\nen/cIyGEypV0r47K+n4+8j1l73PWEXCX99nn7C2qijHGGFNfIW4HYIwxpnmwhGKMMSYgLKEYY4wJ\nCEsoxhhjAsISijHGmICwhGKMMSYgLKEYY4wJCEsoxhhjAsISijHGmIAIczuAhtShQwdNTk52Owxj\njGlSvvrqqwuqmlBTuRaVUJKTk9myZYvbYRhjTJMiIsf8KWe3vIwxxgSEJRRjjDEBYQnFGGNMQFhC\nMcYYExCWUIwxxgSEJRRjjDEB4UpCEZH2IvKRiBx0PuMqKTNJRLaX+ykQkRnOvp4issmp/7qItGr4\nqzDGGFOeWy2UJ4E1qtoXWOOsX0ZV16rqcFUdDkwG8oAPnd2/A/7k1M8AHmuYsI0xxlTFrYRyOzDf\nWZ4PzKih/N3AClXNExHBl2DeqkV9YxqtMa/cxZhX7nI7jIBoTtdy7/NfcO/zX7gdRr015HW49aZ8\nJ1VNBVDVVBHpWEP5WcAfneV4IFNVS5z1k0DX4IRpGqvSfyCvf+calyMxzdXRVv/hLL3tahz11ZDX\nEbSEIiKrgc6V7HqqlsdJBIYAq0o3VVJMq6k/F5gL0L1799qc2jRizeUfuzHNSdASiqpOrWqfiJwV\nkUSndZIInKvmUDOBJapa7KxfANqJSJjTSukGnK4mjheAFwBSUlKqTDzGGGPqx60+lGXAw87yw8DS\nasrOBhaVrqiqAmvx9av4U98YY0wDcCuhPAvcICIHgRucdUQkRUTmlRYSkWQgCVhXof5PgX8WkUP4\n+lReaoCYjTHGVMOVTnlVTQOmVLJ9CzCn3PpRKulwV9XDwOgghmiMMaaW7E15Y4wxAWEJxRhjTEBY\nQjHGGBMQllCMMcYERIuaU96YxuiXC/b6Fh5xN45AsGtpfBryOiyhmCapufxjN6Y5sVtexhhjAsJa\nKMY0U75BJQBV308ly3qpcLVlfR/Odq/30r7LyngJ9fqWSy5UM5qS11tT4DVfXE3HoKb9oDUcI1x9\n+4tPHavhQLUZ0cnPsn4fs+ZyrZzr0KJCpFWEn8etG0soLUyzGKU3L52O7fKJDfGQP+/7qMeDejzg\n8aAeL+rxOsse8HqdbZeWL23zXrnN6wWvOtsU9XpRr4LH+fQq6vxQ9ulFvVTYpr5temkZZ/3SMuBV\nensVVdg3uP+layz/PaGVLFb2PaJQ+dipDad0+NWD4693NY5A6Ob8Lg9Nme5yJPXT1bmOop0biEiZ\nHNRzWUIxjZsqXDgIJzbiPbSB3C82k7s3nTanI/AURXJ054c1H6M2BESAEJAQ8a2HSNm6hAAhgog4\n26Xc/lBfmXDffkIFCQlx9ocgoaWfISCChPqWc44dAqBN737OySn3CVIWlG+9bPmycuJ8VNgnpZV8\ncfvWym2vZD/IpVOIXCpXulz2e7pUvrTuuU3rAKHj2BoSSvk467KfCtdax2NUV+bMpx8B0HnCjTUf\npzb8iat2B6x2b+p630DtYd16Bfi8V7KEYhqXojw4vRVObILjmyg+sJncbwrJORVJ3rkI1COERLej\nICqf7FhlxLMvOl/SYUhYKISGIqHOZ1iYbzkkFAlztpduc8qVbQsJ8W0Lafhuxa1TBwJw0wsrGvzc\ngbbRuZb+v3rF5Ujq7wvnWgY89bzLkdTPFx/7rmNg5+Sgn8sSinFX9umy5MGJTWjqTooyIedUJDln\n21FwNgqIIjyxI+3um0bExEkcSezDgn99ksMJkbQ7eRKvelE8eNSL4sWrHrx4Ua2wjBfw4FWvU8fr\n7POglNb1li2X335pm/eKbeBBUWe5/HZvhW0K4nHWFUr3zYwH4KcvXQ8IaGlL4tJ/qWRZqtl3eW2p\nUEbKlRQqO4dUUafi8Sque25PBhV+OW9mlee9/JyVb6ss7ivrVfX7qPsxL5URCqeOAOB3835U/jQV\nfu+X14SKDZBye65cuGLpsgbmFf+t/DgV46l4NbkTh4Mqvc6l0r9j4hWxB5IlFNNwPCVwdjec2OxL\nIic2QdYJ1Av5GbHkZHYj52gvis/nAhAxaABy83gO9B7BJolj26kzfLN+LRLz34RN2Y+EFkLR+trH\nUfG7qCL13fMS56eqZZHLt4WU+0RKl0MRwhEJvVRGLh0jRELJO3UCUYhJSqa0m7ysQ710nXKd31T8\ndP6rpVsvdbD4li8vX7Z+RfnLj3hF/cs6iivWc8oV5INAaHhY2b7Sn6rjrLhU8bgVfg8Vtl0Zd8V6\nlV9/5dvLbevoASA/ZF+FehX53ymvlZatZQe8P8WlXKEk30dafgZgCcU0VfmZcHKLkzw2wsmvoPgi\nAN7IRHIL+pJ74ipyd53Ak50LYRfJGzSC/dcN5+O4fnyeHUp+djphhz4iss1eJPYbWrXx0DqsHb33\nexh6wsOkf59PiIQQKqG+z5DQS8vOZ5iEERJyab10X2VlQyW05nvzATbzheEAvHFf07/lVTqf/MaH\nm/5MmqXXsunbTftaSq9jXI+BQT+XJZQWJmhT56pC+uFLLY8Tm+HcXkBBQqDTYEp630VOagy5u8+Q\n++U2KDpEcVQMB3oOZU37fqxr24e88AgiIs7Tuc3ndOi8mwzPYQC6x/ZgSveHmNx9MkM6DOHa+TM5\n2BN+0nlUYK/DBcfCe7sdgjEBYQnF1E1xAaRuv5Q8TmyCi+d9+yLaQtIodOAMiuhB2q5zpK1cT+j+\nNYgqF1q357Nuo9mYOIivO/SiZ+dYuieeY0z0dk4UfMnZ/FOkA0PjhvJQ9+8zOWkyPdv2bPCWgzGm\ndiyhtDA9ir+pW8Xcc07n+UZfAkndDp4i3772vaDPVEgaQ1Gnq9m/K50LH64h6n9W0jb9DACH23Vj\nY78bOXzVSDoMHcjAblF8K+YQA/I38/np9WwuzCC8OJzRiaOZm/QoE5Mm0jG6Y4Cu2hjTECyhmCt5\nPXB+36XkcWITZBzx7QttBV1GwJjv4u02muMxQ9h+upjzaz8j6m+f0eebv9K26CLxEsrezn05c8ON\nhE+4nv5DezMnAXZlfMHa43/j1ZOfU+ApIDY8lgndJjC5+2TGdRlH61at3b12Y0ydWUIxUJjjdJ5v\ndjrPt0Bhtm9fTAIkjYGUR0lrP4Kvinuw/XQeh/YeI+rvXzDi+CsMP3+Qft4S8ltFcX5QCvnXT6T3\nzVOZ2S2Bk7knWXt8LX87+gLbNm/Dq146RXdiRp8ZTO4+mZROKYSHhrt7/caYgHAloYhIe+B1IBk4\nCsxU1YwKZSYBfyq3qT8wS1XfFZEngB8AvYEEVb3QEHE3C6okeEpg55u+5HFiE5z9Gt94IAIdB8Lg\nu8jrPIqvQ/uxObMt209msfOTDFqdPsjY1He55szX3Jx+nBCUog6daHXHnXS9aRqxo0dBWBh70vew\n9PjrrN22loMZBwHoG9eXfxjyD0zqPomB7Qdaf4gxzZBbLZQngTWq+qyIPOms/7R8AVVdCwyHsgR0\nCCgdZ2MDsBz4pKECbha8Hp5JS6V/USG8MwdatYZuKZSM+xFHogezsagXX53xsGN/Fkc2XCREUxmQ\nvpHpmft59NRu2jn9Ia0GDKTN/Y8TO2UKEf36UeIt4cuzX7L2q//H2hNrOZt3lhAJ4eqOV/PjlB8z\nqfskkmKTXL54Y0ywuZVQbgcmOsvz8SWGn1ZVGLgbWKGqeQCqug38GMvHXO7kFvoXFbLrbEdO3/VX\nPs1OYNvJXPbvy6HEq8AJukUJtxSfYNTZ3XTes4WQrEwIDydm9GhaT/kHYidNIjwxkdyiXD45/Rkf\nf/oyn538jJziHCJDI7m2y7X804h/4rpu1xEXGef2FRtjGpBbCaWTqqYCqGqqiNT0OM8s4I91OZGI\nzAXmAnTv3r2G0s2bd98HeDWE+9v+huzVxcRGnmNYt3Y8MaI9Kae/pvPuzZRs3oQWFhISG0vr668n\ndvIkYiZMIDQ2lnN551hy4hM+/vpjNqdupthbTFxEHFN7TGVS0iTGdhlLVFiU25dpjHFJ0BKKiKwG\nOley66laHicRGAKsqkscqvoC8AJASkpKbSYuaHYK9nzAdm9/xp3eyI++9xhtt37BxTXzyd++3Tc3\nRpdE2t1zD7FTJhOdkgJhYRzJOsIbR19n7fG17LywE4Ck2CTu638fk7tPZljCMEJDQl2+MmNMYxC0\nhKKqU6vaJyJnRSTRaZ0kAtXMxsNMYImqFgc8yJYk/QjRmQc4sW8C39m7iuIHVnIBiBw4kA5PPE7s\n5MlE9O+PV73svLCTtTv+wscnPuZY9jEABscP5nsjvsekpEn0btfbbjcaY67g1i2vZcDDwLPO59Jq\nys4GftYQQTVnun8FBWnhDN31DXmtQuj0r08RO3ky4YmJFJQUsCl1Ex9/sZhPTnxCekE6YSFhjO48\nmgcHPMjEpIl0iunk9iUYYxo5txLKs8AbIvIYcBy4B0BEUoDvquocZz0Z31iZ68pXFpHvAT/Bd0tt\np4h8UFrHVC5/93KO70mgMDSMEwnh9L7rZlacWs/atWvZcHoD+SX5tA5vzYSuE5jUfRLju44ntlWs\n22EbY5oQVxKKqqYBUyrZvgWYU279KNC1knL/BfxXEENsXgqykH1b8J6KZ1/3MBZMKeFXb0zEox46\nRnfktt63MTlpMqM6j7KXDI0xdWZvyrcEh1aTsTcST2gof5lRTLhHeHTwo0zuPpmB8QMJkYafpdAY\n0/xYQmkBcj9/h6yj0WwblUR29CmitCffu/p7bodljGlmLKE0d55iclZuRLUVi1POE0prpBn8sW96\npGlPelRec7oW07I1/W8WUy3Pno/JPhDG173bc6JNNhFee1rLBE9zSo7N5Voa8jrs5nkzl/bqC3hL\nQnj7ulCGJgwllEi3QzLGNFOWUJoxb0EBGR/vIb1LGF93yuC+/ve5HZIxphmzhNKMZS14EW8+LBwb\nS3xkB27scaPbIRljmjFLKM2UejykzX+NkPbFrL8qm3v7zbR3TIwxQWUJpZnK+Wg1xeeyWZ0SQWhI\nOPf0u8ftkIwxzZw95dUMqSppLzxHWGwJrwyNYnLSVDpEdQBgYGIbl6MzxjRX1kJphvI2baJgzz6O\nDCumMFz59uAH3Q7JGNMCWEJphtJenEdITBh/HRlFp4i+DE0Y6nZIxpgWwBJKM1OwZw8XN2wge0Ae\nh6PDeGCAPSpsjGkYllCambR5LxESFcmiFCHCE8l9g29zOyRjTAthCaUZKTpxguyVK2FUAqvbRdK/\n9XRahbZyOyxjTAthT3k1I+mvvIKEhvJBn5MIoTwy4qEryrwy/RUXIjPGtATWQmkmStLSyHz7HWKm\nXsubHYRueYlM6t3H7bCMMS2IJZRmIv2119CiIr4cnE1OaAhXtbuXkBBxOyxjTAviSkIRkfYi8pGI\nHHQ+4yopM0lEtpf7KRCRGc6+BSKyX0R2i8jLItKixxTxXrxIxsJFtJ4yhVfka3oWwE3DbnY7LGNM\nC+NWC+VJYI2q9gXWOOuXUdW1qjpcVYcDk4E84ENn9wKgPzAEiKLcPPQtUcabb+LNyuLMtwZxNNRL\nUnYvxvdNcDssY0wL41ZCuR2Y7yzPB2bUUP5uYIWq5gGo6gfqADYD3YIWaSOnRUWkvzqf6FGj+Hvh\nKuI8Htq3nUlUq1C3QzPGtDBuJZROqpoK4Hx2rKH8LGBRxY3Ora4HgZUBj7CJyHr/A0rOnEHvn8G6\n7ENMy/YwePh4t8MyxrRAQXtsWERWA50r2fVULY+TiO/W1qpKdv8PsF5VP62m/lxgLkD37t1rc+pG\nT71e0l6aR0S/frwddwA5o7TL7M/kAZX92o0xJriCllBUdWpV+0TkrIgkqmqqkzDOVXOomcASVS2u\ncIxfAgnAd2qI4wXgBYCUlBT1N/6mIPeTdRQd+ob4f/81bx/8f0y5mEda2ykkxEa4HZoxpgVy65bX\nMuBhZ/lhYGk1ZWdT4XaXiMwBpgGzVdUblAibgLR58wjv0oV1/UvI8eRzZ1YRHYdVmceNMSao3Eoo\nzwI3iMhB4AZnHRFJEZF5pYVEJBlIAtZVqP8c0An4wnmk+BcNEXRjkrd1K/lbtxL3yLdZeOB1rir2\nkpnfnymDktwOzRjTQrky9IqqpgFTKtm+hXKPAKvqUaBrJeVa/JAxaS/OI7RdOw6P78mh9Yd4JjOD\nHVF3clPH1m6HZoxpoexN+Sao8OBBcteuJe6BB1hw9C3ahrTixtx8IgZMR8TejjfGuMMSShOU9tLL\nSFQUhTMms/bEWm69qOzx9uHaof3dDs0Y04JZQmliik+fJmv5ctrdfTdvnl2JAA+dP8qGkFGMSr5i\nBBtjjGkwllCamPT580GVmAdn8fbBt5nUuieJHg/5vW4kLNT+OI0x7mnxndtNiSczk4w336LNzTfx\nUeF2sgqzuC03mqPeTgwdNtrt8IwxLZz9L20Tkr5wIZqXR/vHHmPh3oVc1a4P409t5WMdyXX9bDBI\nY4y7LKE0Ed78fDL+/hox11/H121z2J+xn/vaDSFciznTeSKxkS16BH9jTCNgCaWJyHznHTwZGXSY\nM4eF+xbSplUbJpw+SbZGkzR8stvhGWOMJZSmQEtKSH/5FaKGDSNrQDc+Pv4xd/W9k9jDH7PWO5wp\ng1rs6P3GmEbEEkoTkL1yFcWnThH/D3N448AbKMqsNgOIKs5gf5txdGkX5XaIxhhjCaWxU1XS5s2j\nVa9ehF8/jrcPvM3EbhOJO/QFxRpK68HT3Q7RGGMAPxOKiIwTkRhn+QER+aOI9AhuaAbg4mcbKNy3\nj/jHHmXlsVVkFGZw/4D7Kdr7AV96+3Hd0L5uh2iMMYD/LZT/BfJEZBjwE+AY8LegRWXKpM2bR1jH\njsTecgsL9y6kT7s+jGrVgbY5h9jcagyDurRxO0RjjAH8TyglzvzttwP/qar/CcQGLywDkL9rF3mb\nNtH+4YfZmbWHvel7md1/NiX7VgDgvcoGgzTGNB7+JpQcEfkZ8ADwvoiEAvbiQ5ClvTiPkNhY2t07\nk4V7FxLbKpZbet1Czs7lHPR2ZeSIkW6HaIwxZfxNKPcChcBjqnoG3xwlvw9aVIbCI0fI+egj4mbP\n5oJc5KNjH3FnnzuJ9hTT9uwm1kkKY3u1dztMY4wp4+9YXjn4bnV5ROQqoD8VpuU1gZX+8itIeDjt\nH3qQ/z2wCK96ubf/vXgPriYUD5lJU4gIC3U7TGOMKeNvC2U9ECEiXYE1wCPAq8EKqqUrPneOrHff\npe0dd+CJi+WtA29xfdL1JMUmkbV9GWkaS68RE90O0xhjLuNvQhFVzQPuBP6iqncAg4IXVsuW8fe/\nox4P8Y8+wqqjq0gvSOe+/veBp4TIYx+z1juCSf0T3Q7TGGMu43dCEZFrgPuB951tdb7fIiLtReQj\nETnofF4xM5SITBKR7eV+CkRkhrPvJRHZISI7ReQtEWk2E6l7cnLIWLSY2Gk3Et69Owv2LqBX216M\nTRwLJzYSVZLNsfjriItp5XaoxhhzGX8Tyg+AnwFLVPVrEekFrK3HeZ8E1qhqX3y30J6sWEBV16rq\ncFUdDkwG8oAPnd0/VNVhqjoUOA48UY9YGpXM11/Hm5tL/Jw57Di/gz1pe7iv/32ICDk7llGoYXQY\nZm/HG2MaH78SiqquU9XbgP8RkdaqelhVv1eP894OzHeW5wMzaih/N7DCue2GqmaDr9kERAFaj1ga\nDW9REenz/0bMtdcQNWgQC/ctJDY8llt73+rbv38FG70DuW5IL5cjNcaYK/k79MoQEdkG7Ab2iMhX\nIlKfPpROqpoK4Hx2rKH8LCo8VSYirwBn8D1x9pdqYp8rIltEZMv58+frEXLwZS1dSsn588TPmcO5\nvHN8dPQjZvSdQXR4NFw4SNu84+yIuYaeHWLcDtUYY67g7y2v54F/VtUeqtod+BHwYnUVRGS1iOyu\n5Of22gQoIonAEGBV+e2q+gjQBdiL7z2ZSqnqC6qaoqopCQmNd1ZD9XhIf+llIgcOJPqaa3jzwJt4\n1MPsfrMByN/9HgBh/W9yM0xjjKmSv++hxKhqWZ+Jqn5SOlhkVVR1alX7ROSsiCSqaqqTMM5Vc6iZ\n+Ppuiis5h0dEXgd+DLxS41U0Yjlr1lB09Chd//RHSrwlvLn/TSZ0m0BSmyQA8nYt54i3B2NGDHU5\nUmOMqZy/LZTDIvKvIpLs/PwLcKQe510GPOwsPwwsrabsbMrd7hKfPqXLwK3AvnrE4jrfEPUvEd69\nO7E33siqY6tIK0jj/v73+wrkpROXto3PQ1MYnnTFA3HGGNMo+JtQHgUSgHeAJc7yI/U477PADSJy\nELjBWUdEUkRkXmkhEUkGkoB15eoKMF9EdgG7gETgmXrE4rq8zV9SsHMn8Y8+goSGsnDvQpLbJDO2\ny1gASvavJAQveT1vJDTEBoM0xjROft3yUtUMoD5PdVU8XhowpZLtW4A55daP4hs3rHwZLzAuULE0\nBmnz5hEaH0/bGTPYdX4Xuy7s4mejf0aI+PJ95rb38Go7+l99ncuRGmNM1apNKCLyHtU8kus8Smzq\noWDfPi5++ikJP/gBIZGRLPxyITHhMdzex3l2oaSI2JOf8K6O5daranoYzhhj3FNTC+U/GiSKFixt\n3kuEREcTN3sWF/IvsPLoSu7tdy8x4b5nHvToZ0R48zjbeRLRrfx9hsIYYxpetd9Qqrquuv2mfopO\nniJ7xQraP/QQoW3b8uaO/6XEW8KsfrPKymRsX0a0htN5xDQXIzXGmJr59b+8Tgd4xVtfWcAW4DdO\nn4ippfRXXoGQENp/+2GKPcW8uf9NxncdT3LbZF8BVcIOruRT7xAmDurhaqzGGFMTf5/yWoFvUMj7\nnZ/3gE/xvan+alAia+ZK0tPJfPtt2t56K+GdOvHRsY84n3/eN6pwqXN7aFOYyoG24+nYJtK9YI0x\nxg/+3pQfp6rln6zaJSIbVHWciDwQjMCau4zXFqAFBcQ/9igAC/ctpEebHozreunXnLtzGa2B6MH2\ndrwxpvHzt4XSWkTGlK6IyGigdMj4koBH1cx58/LIWLCA1lOmENG7N19f+Jod53cwu//sskeFAQp2\nf8B2by+uHT7YxWiNMcY//rZQ5gAvO/OOCJANPOYMv/LvwQquucp86y08WVnEz3kM8LVOosOiub13\nuWHOcs7SPmsX70bM4rFOzWa6F2NMM+bvi41fAkNEpC2+2Rszy+1+IyiRNVNaXEzaq68SlTKS6BEj\nSMtPY8WRFdzV9y5at7qUOAr3riACpaTPdHwjzBhjTOPm7/D1bUXkj/gmw1otIn9wkouppewPPqDk\ndCrxc3wDArx14C2KvcXMHjD7snKZ25dxSuMZenWzGhTAGNOM+duH8jKQg2/k35n4bnk16dF93VA6\nCGRE3760vv56ir3FvLH/Da7tci292pabNKs4n7jUz1gvKYzqFe9ewMYYUwv+JpTeqvpLZ6bGw6r6\nNGDTBtZS7rp1FB48SPycxxAR1hxfw7n8c9w/4P7Lynm+WUcrLSSz2xTCQ/39IzLGGHf5+22VLyLj\nS1dEZByQH5yQmq+0efMI65JIm5t8jwEv3LuQpNgkxncdf3m5rUvJ1Ui6X21vxxtjmg5/n/L6P/iG\njG+L7ymvdODbwQqqOcrbto38LV/R6ec/Q8LD2Zu2l23ntvHjlB9f9qgwqkQd+YjPdCgTBnat+oDG\nGNPI+PuU13ZgmIi0cdazgxpVM5Q27yVC27al3d13A75HhaPCopjRd8blBVO3E1t8nmPxDzM9MtyF\nSI0xpm5qGr7+n6vYDoCq/jEIMTU7hd98Q+6aNXT4x38kJDqa9IJ0Pjj8AXf0vYM2rdpcVjZj2zLa\nqNBmqL0db4xpWmpqocQ2SBTNXNpLLyORkcQ96Bul5p2D71DkLWJ2/9lXlPXs/YCt2pcJwwc0dJjG\nGFMvNQ1f/3RDBdJcFZ85Q9Z77xE3cyZhcXGUeEtYvG8xYxPH0rtd78sLZ52iQ+4+lsV8m1Fx0e4E\nbIwxdVTrZ1JFZGt9Tyoi7UXkIxE56HzGVVJmkohsL/dTICIzKpT5i4jk1jeeYEqf/zfwemn/yCMA\nfHz8Y87mnb18VGHHxV3LAQjp960GjdEYYwKhLi85BGIckCeBNaraF9/b909WLKCqa1V1uKoOByYD\necCHZUGIpADtAhBL0Hiyssh8/XXafOtbtOrme2Jr4b6FdG3dleu6XTk/fM7O9zjq7cTVI8dcsc8Y\nYxq7uiSU9wNw3tuB+c7yfGBGNWUB7gZWqGoegIiEAr8HfhKAWIImY9FivHl5ZYNA7k/fz1dnv2J2\n/9mEhoReXrgwl/jzG/k8bBSDuzbqPGmMMZWqdUJR1X8JwHk7qWqqc7xUoGMN5WcBi8qtPwEsKz1G\nY+QtKCD9738nZsIEIvv3B8o9KtznyvxZdGAN4VpMXs8bCQmxwSCNMU2Pv1MA51D1FMA/UtXDldRZ\nDXSu5HBP1SZAEUkEhgCrnPUuwD3ARD/rzwXmAnTv3r02p66XrHffxZOWVjYIZGZBJu8ffp9be99K\n24grx9VM27qUGI2mT8rUBovRGGMCyd835f8InAYW4utDmYUvWezHN3DkxIoVVLXKb0YROSsiiaqa\n6iSMc9WceyawRFWLnfURQB/gkPM+TLSIHFLVPpVVVtUXgBcAUlJSKibFoFCPh7SXXyFy6FCiR48C\n4O2Db1PoKay0Mx6vh9gTa/iEEUztU1kONsaYxs/fW17TVfV5Vc1R1WznS/omVX0duOIJLT8sAx52\nlh8GllZTdjblbnep6vuq2llVk1U1GcirKpm4JefDDyk+frxsEMgSbwmv73+d0Z1H0zeu7xXl9eQW\nWpdkcqbTRCLDQys5ojHGNH7+JhSviMwUkRDnZ2a5fXX5v/5ngRtE5CBwg7OOiKSIyLzSQiKSDCQB\n6+pwDleoKmkvzqNVcjKxU6YEg+6AAAAb6UlEQVQAsO7EOlIvplbeOgHOf7WUYg0lYcTNDRmqMcYE\nlL+3vO4H/hP4H3wJZCPwgIhE4esgrxVVTQOmVLJ9C77phkvXjwLVjpCoqo1qfty8L76gYM8eOv/6\nGSTU19pYsG8BXWK6cH3S9ZXWCTm4gi+1H+OHNKqGljHG1Iq/g0MeBm6tYvdngQun6UubN4+whATa\n3u6bH/5AxgG+PPMlPxz5Q8JCKvl1px+hQ95hlredy7WtIxo4WmOMCRx/pwC+SkTWiMhuZ32oiATi\n8eFmJX/311z8/AvaP/wQIa1aAbBo3yIiQyO5q+9dldbJ3L4MgIhBdrvLGNO0+duH8iLwM6AYQFV3\n4nvSy5ST9tI8Qlq3pt299wKQVZjF8m+Wc3Ovmyt9VBggb/f7HPR2ZczIlIYM1RhjAs7fhBKtqpsr\nbCsJdDBNWdHx4+Ss+pC42bMIjfUN0rzk4BIKPAWVjioMQEEWHdO3sCVyDL0SGlVXkDHG1Jq/CeWC\niPTGeaJLRO4GGu1b6m5Ie/llJDSUuAcfBMDj9bB4/2JSOqXQr32/Suvk7V1FGB6Ke9tUv8aYps/f\np7wex/dyYH8ROQUcwffklwFKLlwg650ltJ0xg/COvlFk1p1cx6ncU/wo5UdV1kvfuox8jWXAqCse\neDPGmCbH34RyCngFWAu0B7LxvZD4TJDialLS//4aWlxM+0cfKdu2cN9COsd0ZlLSpMoreUqIO/UJ\na+Rqbk7u0ECRGmNM8Ph7y2spvseGi/ENwZILXAxWUE2JJ/ciGYsWEXvDDUT07AnAN5nfsCl1E/f2\nu7fyR4WBkmNfEOPNIb3bFEJtMEhjTDPgbwulm6pOD2okTVTmG2/gzc4m/h/K3sdk4d6FtAppVeWj\nwgBntyyhg4bRdaQ9LmyMaR78baF8LiJDghpJE6RFRaTPn0/0mDFEDfH9erKLsnnv8Hvc1Osm4iKr\nHuYs8psP2aSDuHZgcgNFa4wxweVvQhkPfCUi+0Vkp4jsEpGdwQysKch6bzklZ8+WDVEP8O7Bd8kv\nya9y3C4APX+A+MITHOswgZgIfxuJxhjTuPn7bWaTnFegXi9pL71ExIABxIwfB/geFV60bxFXd7ya\nAfEDqqx7YetSEoDWQ6sazcYYY5oef8fyOhbsQJqa3LVrKTp8mC7/8R8487Lw2anPOJl7ku+P/H61\ndYv3vM8ebw+uGTG8IUI1xpgGUZc55Vu80iHqw7t1o830Sy8lLty3kI7RHZnSvZr3SvLS6ZS1g10x\n19C5bWQDRGuMMQ3DEkod5H/1Ffnbt9P+kW8jYb5G3uGsw3x++nPu7Xcv4SHhVdbN3vU+oXihn91F\nNMY0L9YjXAdpL84jNC6OdnfeWbZt0d5FhIeEV/uoMEDmtmUUaDuGjp4Y5CiNMaZhWQullgoOHCB3\n3TriHnyAkKgoAHKKclj6zVK+1fNbxEfFV125pIiEs5/xRdgo+idWPvqwMcY0VZZQain9pZeQ6Gja\n33fpseClh5b6HhUeUPWjwgCF36wnSvO42OOGso58Y4xpLiyh1ELx6dNkvf8BcffcTWi7dgB41cui\nfYsYnjCcQfGDqq1/9st3KdBwkkfd1BDhGmNMg3IloYhIexH5SEQOOp9XvFIuIpNEZHu5nwIRmeHs\ne1VEjpTb1yDP36a9+ioA7b/97bJtn536jOM5x2tsnaBK62Mf8QVDSenbNXhBGmOMS9xqoTwJrFHV\nvsAaZ/0yqrpWVYer6nBgMpAHfFiuyI9L96vq9mAHXJKRQeabb9H25psJT0ws275w30ISohKY2mNq\ntfU9Z76mffEZznSeRKswaxgaY5oft77ZbgfmO8vzgRk1lL8bWKGqeUGNqhoZCxei+fnEz3msbNvR\nrKNsOLWBmf1mVvuoMMCZL98FIG6EvR1vjGme3EoonVQ1FcD57FhD+VnAogrb/s0ZV+xPIhJRVUUR\nmSsiW0Rky/nz5+sU7IffGsXZv/6V1hMnEtG3b9n2Rft8jwrffdXdNR5DDnzADm9vrhlafT+LMcY0\nVUF7D0VEVgOdK9n1VC2PkwgMAVaV2/wz4AzQCt9Mkj+lism+VPUFpwwpKSlam3OXiskpJtSrlw1R\nf7H4Iku/Wcq05Gl0iKphgqzcc3TO3cOGtg8yLLr6lowxJjiKi4s5efIkBQUFbofSaEVGRtKtWzfC\nw+v2PRW0hKKqVXYqiMhZEUlU1VQnYZyr5lAzgSWqWlzu2KXz2ReKyCvA/w1I0FWIulhCYWQI0SNH\nlm1bemgpF4svcv+AmmdCvrB1GR1Qwgfa3CfGuOXkyZPExsaSnJxsj+1XQlVJS0vj5MmT9HQmC6wt\nt255LcM3hTDO59Jqys6mwu0uJwkhvr8VM4DdQYixzPnEKM53iipbL31UeGiHoQzuMLjG+rk73+OU\nxjNy9IRghmmMqUZBQQHx8fGWTKogIsTHx9erBedWQnkWuEFEDgI3OOuISIqIzCstJCLJQBKwrkL9\nBSKyC9gFdAB+E9RoRfCWezLri9NfcDT7KLMHzK65bnE+ndO+4KuIsSTFxwQxSGNMTWqbTO59/gvu\nff6LIEXT+NQ32bqSUFQ1TVWnqGpf5zPd2b5FVeeUK3dUVbuqqrdC/cmqOkRVB6vqA6qaG8x4F//T\nIBb/06XO9IX7FtIhqgPTekyrppZPzt41RGohRb1vDGaIxpgmoHXr1mXL06dPp127dtxyyy2Vln38\n8ccZPnw4AwcOJCoqiuHDhzN8+HDeeuutWp1z69atrFy5sl5x+8sGh6yl49nH+fTkp3x32HcJD625\n4+rC1qWIRtJ3jI0ubIy55Mc//jF5eXk8//zzle7/61//CsDRo0e55ZZb2L69bq/bbd26ld27dzN9\n+vQ6x+ove8OulhbtW0SohHLPVffUXFiVuBMfsylkOEO61/RktDGmJZkyZQqxsbF1qnvw4EGmTZvG\nyJEjue666zhw4AAAixcvZvDgwQwbNoxJkyaRn5/PM888w4IFC+rUuqkta6HUQl5xHu8eepcbk28k\nITqhxvJFJ7fSznOB9G5zCQmxjkBjGoun3/uaPaezayy3J9VXxp9+lIFd2vDLWxvmPbO5c+cyb948\nevfuzYYNG3jiiSf48MMPefrpp/nkk0/o1KkTmZmZREVF8Ytf/ILdu3fz5z//OehxWUKphWXfLCO3\nOLfmcbscpzctIUmFTiNvC3JkxpiWIjMzk40bN3LXXZfmXiopKQFg3LhxPPTQQ9xzzz3cWW6+poZi\nCcVPqsrCfQsZFD+IoR2G+lWn1Ter2M5VjB58VZCjM8bUhr8tidKWyevfuSaY4dSKqtKhQ4dK+1Re\nfPFFNm3axPLlyxk2bBg7d+5s0NisD8VP2UXZHMk6wv0D7vfr0TrNOkmX/AMcjZ9AZHhoA0RojGkJ\n4uLiSExMZMmSJQB4vV527NgBwOHDhxk7diy//vWviYuL49SpU8TGxpKTk9MgsVlC8dO5vHO0j2zP\ntOSaHxUGOL3ZNxhk9BAbDNIYc6UJEyZwzz33sGbNGrp168aqVatqruRYvHgxzz33HMOGDWPQoEEs\nX74cgB/+8IcMGTKEIUOGMHXqVAYPHszkyZPZsWMHI0aMsE75xqCwpJCsoiy+M/Q7tApt5Vedoj3v\nc1Q7MSplbJCjM8Y0Fbm5l16Z+/TTT/2qk5yczO7dlw8G0qtXr0oT0LJly67YlpCQwJYtW2oZad1Y\nQvHDufxzCMLMfjP9q1B0ka4ZX7Iq5hZujY0MbnDGmKBpTH0nTYHd8vJT+8j2dIz2712S9J2raEUx\n3r7Bf5HIGGMaC2uh+CEpNglV/0e+T9/2LmEazaCx/vW3GGNMc2AtFD/5PWia10PH1HV8GZ5C787t\nghuUMcY0IpZQAuzikU208WaS3X2KDZNtjGlRLKEEWOrmJRRrKEmj7O14Y5q8V272/Ri/WEIJsJij\nH7FN+jP8qmS3QzHGNDINPXz9kiVL+P3vf1/vuP1lnfIBVHLhCImFR9jU8XFGh1quNsZULVDD15eU\nlBAWVvlX+R133BGYYP1k33oBdHLTOwDEjbjd5UiMMY1dfYavHz9+PE899RTXXXcd//3f/83SpUsZ\nM2YMI0aM4MYbb+TcuXMAzJs3jx/84AcAPPDAA3z/+9/n2muvpVevXmVDtwSStVACSPd9wCHtysir\nR7odijGmOiuehDO7ai53xhlc0Z9+lM5D4FvP1i+uWsjOzmb9+vUAZGRkcNtttyEiPPfcc/zhD3/g\nd7/73RV1zp07x4YNG9i1axczZ84MeAvGlYQiIu2B14Fk4CgwU1UzKpSZBPyp3Kb+wCxVfVd8j0/9\nBrgH8AD/q6r/1QChV0nzM0nK2cbKtnfTJ8LytDEmuGbNmlW2fPz4cWbOnMmZM2coLCzkqqsqH+F8\nxowZiAhDhw7l1KlTAY/JrW++J4E1qvqsiDzprP+0fAFVXQsMh7IEdAj40Nn9bSAJ6K+qXhFxfTrE\ns9s+oDMewgbc5HYoxpia+NuSKG2ZPPJ+8GKpo5iYmLLlxx9/nJ///OfcdNNNrF69mmefrfz6IiIi\nypZr87K2v9zqQ7kdmO8szwdm1FD+bmCFquY56/8HeEZVvQCqei4oUdZC9vZlpGksw8ZOdTsUY0wL\nk5WVRdeuXVFV5s+fX3OFIHEroXRS1VQA57OmFsYsYFG59d7AvSKyRURWiEjfIMXpH08Jiec/ZVvk\naBLjWtdc3hjT4tVn+PqKfvWrX3HHHXdw/fXX06lTpwBGWTtBu+UlIquBzpXseqqWx0kEhgDlf9sR\nQIGqpojIncDLwIQq6s8F5gJ07969Nqf2W+aBz2inuRT2vDEoxzfGNA+BGr7+s88+u2z9rrvuumxK\n4FJz5swpW37ttdeqjCVQgpZQVLXKez8iclZEElU11UkY1d2ymgksUdXicttOAm87y0uAV6qJ4wXg\nBYCUlJTA3zQEzm5+hygNo9dYm0zLmGalEfadNGZudcovAx4GnnU+l1ZTdjbwswrb3gUm42uZXA8c\nCEKMZV6ZXmW+AqDtyTVsCxnMmB5dghmGMcY0am71oTwL3CAiB4EbnHVEJEVE5pUWEpFkfE9zrauk\n/l0isgv4d2AOLik8s4/OxSe50HWyDQZpjGnRXGmhqGoaMKWS7VsolxxU9SjQtZJymUCjGLHt2Ofv\ncBWQMLKmB9WMMaZ5s6FX6insm5Xs0x4MHzLY7VCMMcZVllDqwZubRo+LuzgSfx0RYaFuh2OMCbBH\nVj7CIysfcTuMJsMSSj0c37yUULxEDa58+GljjCmvdPj67du3c8011zBo0CCGDh3K66+/fkXZQAxf\nD7B161ZWrlwZkPhrYoNO1UPB18s5p+0YPmai26EYY5qQ6Oho/va3v9G3b19Onz7NyJEjmTZtGu3a\nXZo23N/h62uydetWdu/ezfTp0wMSe3WshVJXJUUkpX3OrphraBcT6XY0xpgm5KqrrqJvX98AH126\ndKFjx46cP3/e7/oHDx5k2rRpjBw5kuuuu44DB3xvTixevJjBgwczbNgwJk2aRH5+Ps888wwLFiyo\nU+umtqyFUkdnd31MJ/Lx9JnmdijGmFr63ebfsS99X43lSsv404/Sv31/fjr6pzWWq2jz5s0UFRXR\nu3dvv+vMnTuXefPm0bt3bzZs2MATTzzBhx9+yNNPP80nn3xCp06dyMzMJCoqil/84hfs3r2bP//5\nz7WOrbYsodTRha3v0lbD6X+t9Z8YY+omNTWVBx98kPnz5xMS4t8No8zMTDZu3HjZUCslJSUAjBs3\njoceeoh77rmHO++8MygxV8cSSl2oknD6Y7aFj+CazgluR2OMqSV/WxKlLZOaRsuoi+zsbG6++WZ+\n85vfMHbsWL/rqSodOnSotE/lxRdfZNOmTSxfvpxhw4axc+fOQIZcI+tDqYOc4zvp6DlLVtIV72Ya\nY0yNioqKuOOOO8paE7URFxdHYmJi2RS+Xq+XHTt2AHD48GHGjh3Lr3/9a+Li4jh16hSxsbHk5OQE\n/BoqYwmlDk5s9I1L2WVMYKfPNMa0DG+88Qbr16/n1VdfLXscuDZPcS1evJjnnnuOYcOGMWjQIJYv\nXw7AD3/4Q4YMGcKQIUOYOnUqgwcPZvLkyezYsYMRI0ZYp3xjFHXkI76mD4OrmGbTGGMqUzpk/AMP\nPMADDzzgV53Khq/v1atXpfOnLFu27IptCQkJbNmypQ7R1p4llFoqyjxDj/y9rO70KINCbDBIY5qz\nYPSdNGd2y6uWjm58hxBRYoff5nYoxhjTqFhCqSXP3hWc1niGjxzvdijGGNOoWEKpBS3OJzlrE/va\njCcqwu4WGmNMeZZQauHEV6uIopDQAd9yOxRjjGl0LKHUQtb2peRqJAOvaRRzexljguzYgw9x7MGH\n3A6jybCE4i9VEs+tY2fkSBLi2rgdjTGmCWro4euXLFnC73//+4DFXxPrCPDT+YObSfCmsT35BrdD\nMcY0cYEcvr6kpISwsMq/yu+4o2FfvnYloYhIe+B1IBk4CsxU1YwKZSYBfyq3qT8wS1XfFZFPgVhn\ne0dgs6oGdVL31E3v0F6FXtfY2/HGmPq5qtxL0eWHry+fUKozfvx4rr/+ej799FPuvPNOevbsyW9/\n+1uKiopISEjgtddeo2PHjsybN69spOEHHniA+Ph4vvzyS86cOcMf/vCHgCcct1ooTwJrVPVZEXnS\nWb9stDZVXQsMh7IEdAj40Nk3obSciLwNLA12wG1OrOHr0P4M6dEj2KcyxgTZmd/+lsK9NQ9fX7DP\nV8affpSIAf3p/POf1zqWugxfD77BJdevXw9ARkYGt912GyLCc889xx/+8Ad+97vfXVHn3LlzbNiw\ngV27djFz5sxmk1BuByY6y/OBT6iQUCq4G1ihqnnlN4pILDAZCOqkz3nPTyO56CBruv4jIvZ2vDEm\nMOoyfH2pWbNmlS0fP36cmTNncubMGQoLCy9rAZU3Y8YMRIShQ4dy6tSpesVeGbcSSidVTQVQ1VQR\n6VhD+VnAHyvZfge+lk52VRVFZC4wF6B79+51Cvar1EImAPEjg3pXzRjTQPxtSZS2THr8/W8Bj6Gu\nw9eXiomJKVt+/PHH+fnPf85NN93E6tWrefbZZyutExERUbasqrUPugZBSygishroXMmup2p5nERg\nCHDlSGgwG5hXXX1VfQF4ASAlJaVOv8G2ms1x7cigoSl1qW6MMZepz/D1lcnKyqJr166oKvPnzw9A\nhHUTtISiqlOr2iciZ0Uk0WmdJALnqjnUTGCJqhZXOEY8MBpfKyWovmIAIXh5OCw02KcyxrQApcPX\np6Wl8eqrrwKUDWVfF7/61a+444476NatG6NHjyY1NTWA0fpPgtHsqfGkIr8H0sp1yrdX1Z9UUXYj\n8DOnk7789u8C16jqw/6eNyUlResyjPPXv/WN2zXo55/Vuq4xpnHYu3cvAwYMqFWdYN7yaqwq+z2J\nyFeqWuMtGrf6UJ4F3hCRx4DjwD0AIpICfFdV5zjryUASsK6SY8xyjhN0lkiMaZlaUiIJBFcSiqqm\nAVfMn6uqW4A55daPAl2rOMbEIIVnjDGmDmzoFWOMMQFhCcUY02K40WfclNT392MJxRjTIkRGRpKW\nlmZJpQqqSlpaGpGRkXU+hg0OaYxpEbp168bJkyc5f/6826E0WpGRkXTr1q3O9S2hGGNahPDwcHr2\n7Ol2GM2a3fIyxhgTEJZQjDHGBIQlFGOMMQHhytArbhGR88CxOlbvAFwIYDhuai7X0lyuA+xaGqvm\nci31vY4eqppQU6EWlVDqQ0S2+DOWTVPQXK6luVwH2LU0Vs3lWhrqOuyWlzHGmICwhGKMMSYgLKH4\n7wW3Awig5nItzeU6wK6lsWou19Ig12F9KMYYYwLCWijGGGMCwhJKLYjIr0Vkp4hsF5EPRaSL2zHV\nlYj8XkT2OdezRETauR1TXYjIPSLytYh4nQnamhwRmS4i+0XkkDODaZMkIi+LyDkR2e12LPUhIkki\nslZE9jp/t77vdkx1JSKRIrJZRHY41/J0UM9nt7z8JyJtVDXbWf4eMFBVv+tyWHUiIjcCH6tqiYj8\nDkBVf+pyWLUmIgMAL/A88H+dSdqaDBEJBQ4ANwAngS+B2aq6x9XA6kBErgNygb+p6mC346krEUkE\nElV1q4jEAl8BM5ron4kAMaqaKyLhwGfA91V1YzDOZy2UWihNJo4YoMlmY1X9UFVLnNWNQN2HGHWR\nqu5V1f1ux1EPo4FDqnpYVYuAxcDtLsdUJ6q6Hkh3O476UtVUVd3qLOcAe6li5tjGTn1yndVw5ydo\n31uWUGpJRP5NRE4A9wO/cDueAHkUWOF2EC1UV+BEufWTNNEvr+ZIRJKBEcAmdyOpOxEJFZHtwDng\nI1UN2rVYQqlARFaLyO5Kfm4HUNWnVDUJWAA84W601avpWpwyTwEl+K6nUfLnOpowqWRbk235Nici\n0hp4G/hBhbsTTYqqelR1OL67EKNFJGi3I20+lApUdaqfRRcC7wO/DGI49VLTtYjIw8AtwBRtxJ1p\ntfgzaYpOAknl1rsBp12KxTic/oa3gQWq+o7b8QSCqmaKyCfAdCAoD05YC6UWRKRvudXbgH1uxVJf\nIjId+Clwm6rmuR1PC/Yl0FdEeopIK2AWsMzlmFo0pyP7JWCvqv7R7XjqQ0QSSp/gFJEoYCpB/N6y\np7xqQUTeBvrhe6roGPBdVT3lblR1IyKHgAggzdm0sSk+sSYidwB/ARKATGC7qk5zN6raEZGbgD8D\nocDLqvpvLodUJyKyCJiIb2Tbs8AvVfUlV4OqAxEZD3wK7ML3bx3g56r6gXtR1Y2IDAXm4/u7FQK8\noarPBO18llCMMcYEgt3yMsYYExCWUIwxxgSEJRRjjDEBYQnFGGNMQFhCMcYYExCWUIwJIBHJrblU\ntfXfEpFeznJrEXleRL5xRopdLyJjRKSVs2wvJptGxRKKMY2EiAwCQlX1sLNpHr7BFvuq6iDg20AH\nZxDJNcC9rgRqTBUsoRgTBOLze2fMsV0icq+zPURE/sdpcSwXkQ9E5G6n2v3AUqdcb2AM8C+q6gVw\nRiR+3yn7rlPemEbDmszGBMedwHBgGL43x78UkfXAOCAZGAJ0xDc0+stOnXHAImd5EL63/j1VHH83\nMCookRtTR9ZCMSY4xgOLnJFezwLr8CWA8cCbqupV1TPA2nJ1EoHz/hzcSTRFzgRQxjQKllCMCY7K\nhqWvbjtAPhDpLH8NDBOR6v6NRgAFdYjNmKCwhGJMcKwH7nUmN0oArgM245uC9S6nL6UTvsEUS+0F\n+gCo6jfAFuBpZ/RbRKRv6RwwIhIPnFfV4oa6IGNqYgnFmOBYAuwEdgAfAz9xbnG9jW8OlN3A8/hm\nAsxy6rzP5QlmDtAZOCQiu4AXuTRXyiSgyY1+a5o3G23YmAYmIq1VNddpZWwGxqnqGWe+irXOelWd\n8aXHeAf4marub4CQjfGLPeVlTMNb7kx61Ar4tdNyQVXzReSX+OaUP15VZWcirnctmZjGxlooxhhj\nAsL6UIwxxgSEJRRjjDEBYQnFGGNMQFhCMcYYExCWUIwxxgSEJRRjjDEB8f8BfadWdrYrzP8AAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xf021908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'neg-logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.24282081, -0.24281653, -0.24281947, -0.24282802],\n",
       "        [-0.23781094, -0.2378109 , -0.23781516, -0.23781569],\n",
       "        [-0.23873299, -0.23873138, -0.238735  , -0.23874645],\n",
       "        [-0.23184052, -0.23181967, -0.23180933, -0.23180749],\n",
       "        [-0.24161807, -0.24177731, -0.24185615, -0.24186186]]),\n",
       " 1: array([[-0.50495089, -0.50495176, -0.50495173, -0.50494861],\n",
       "        [-0.49689379, -0.49697611, -0.49691701, -0.49692218],\n",
       "        [-0.49635661, -0.49634281, -0.49632704, -0.49635307],\n",
       "        [-0.50067464, -0.50068016, -0.50071146, -0.50071144],\n",
       "        [-0.50127444, -0.50126243, -0.5012725 , -0.50126071]]),\n",
       " 2: array([[-0.55001195, -0.54998182, -0.54995998, -0.54996878],\n",
       "        [-0.53591964, -0.53602326, -0.5360913 , -0.53600741],\n",
       "        [-0.53738231, -0.53739057, -0.53735264, -0.53736878],\n",
       "        [-0.53946824, -0.53946623, -0.53945665, -0.53945518],\n",
       "        [-0.54817445, -0.54825242, -0.54827753, -0.54835741]])}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAERCAYAAAA0S9PzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VdW5//HPQyDMU0IUZUqAIIJi\n1Bi0Dli1FjtIf5YKnRyurT9/1Wu19lZpvVdr7a22t6L3VmutemvtbQPaqtTWerXOrQJBwywQBiEM\nEkjCTMjw/P7YCxrDCTlATs45yff9euX1OmfvtdZ+dg6cJ2vvtdcyd0dERCTZOiU7ABEREVBCEhGR\nFKGEJCIiKUEJSUREUoISkoiIpAQlJBERSQlKSAlgZt82MzezATH2FZjZ22a22MwWmNmURvvMzH5o\nZsvNbKmZ3Ri29zezZ0L5OWZ2UivEeJ6ZvWtmdWY2+WjbExE5WkpIR8jMzjezX8XYPgT4BLC2maq7\ngSvcfSwwEbjfzPqFfVcBQ4DR7n4iUBy2fxcodfdxwBXAA61wCmvD8X7bCm2JiBw1JaTWNx34DhDz\niWN3X+7uK8LrDcBmICfs/n/AXe7eEPZvDtvHAH8N294Hcs3sWAAz+0roNZWa2S/MLCOeIN19jbsv\nABqO5CRFRFqbElIrMrNLgfXuPj/O8kVAJrAybBoBTDGzEjN7wczyw/b5wGWN6gwDBpvZicAU4Gx3\nLwDqgS+32gmJiLShzskOIN2Y2WygK9ALyDKz0rDrDqJLaxfH2c5xwJPAlft7RKHdve5eaGaXAY8D\n5wL3AA+EYy0E3gPqgAuB04G5ZgbQnajHhZn9GjgtxqEfcveHDuukRUTagGkuuyNjZucDV7n7VeH9\nyUSX1XaHIoOBDUCRu29qUrcP8BrwI3d/qtH294GJ7r7GogxT7e59m9Q1YDUwDrgSON7dpx3FefwK\neN7dnz7SNkREWoMu2bUSd1/o7se4e6675wLlwGkxklEm8Azw68bJKHgWuCC8ngAsD3X6hXoAXwPe\ncPftRAlwspkdE8plmdmwBJyeiEjCKSG1ATMrNLNHw9vLgfOAq8JAhFIzKwj77gE+b2YLgR8RJR+A\nE4HFoQd1CfBNAHdfAtwO/K+ZLQBeAo6LM6YzzKwc+ALwCzNbfNQnKiJyFHTJTkREUoJ6SCIikhI0\nyu4wDBgwwHNzc5MdhohIWpk3b94Wd89pqZwS0mHIzc2lpKQk2WGIiKQVM/sgnnK6ZCciIilBCUlE\nRFKCEpKIiKQEJSQREUkJSkgiIpISlJBERCQlKCGJiEhK0HNIIiISk7uzsmIXc1ZXAvCl8UMTejwl\nJBERAaC+wXl/03bmrK488LN11z4ATh3aTwlJREQSo7a+gUXrtzE7JJ+5ayrZsbcOgMH9uzPhhBzG\n52VRlJdNbnaPhMeT0IRkZhOBB4AM4FF3v6fJ/q7Ar4lWPd0KTHH3NWHfNOAaomW5b3T3Fw/Vppnl\nAcVAFvAu8FV33xf2XQ7cCTgw392/FLZfSbR8A8Dd7v5E6/8WRERSw97aekrXVR/o/cz7oIo9tfUA\nDM/pyWfGHc/4vCzOyMtiUL/ubR5fwhKSmWUADwKfIFqsbq6ZzQpr+Ox3DVDl7iPNbCpwLzDFzMYA\nU4GxwPHAy2Y2KtRprs17genuXmxmD4e2f25m+cA04Gx3r2q8mB3RsuOFRIlqXmirKlG/ExGRtrSz\npo53P6hi9uqtzFldyfx129hX34AZjB7YhylnDKEoL4szcrPI6d012eEmtIdUBJS5+yoAMysGJgGN\nE9Ikop4LwNPAz8IS3ZOAYnevAVabWVloj1htmtlSopVWvxTKPBHa/TnwdeDB/YnG3TeHMp8EXnL3\nytDWS8BE4Het9QsQEWlL1bv3MXdNFXNCAlq0YTv1DU5GJ+PkQX25+uxcivKyKByWRd8eXZId7kES\nmZAGAesavS8HxjdXxt3rzGwbkB22v9Ok7qDwOlab2UC1u9fFKD8KwMz+RnSZ7053/0sz8Q1CRCRN\nbN6xl7mr/9EDen/TDgAyO3eiYEg/vnH+CIrysjhtaH96dk39IQOJjNBibGu6PG1zZZrbHuu5qUOV\nh+gc84HzgcHAm2Z2UpzxYWbXAtcCDB2a2BEmIiKHUl61+yMj4FZt2QVAj8wMTh/Wn8+MO46ivGzG\nDe5Lty4ZSY728CUyIZUDQxq9HwxsaKZMuZl1BvoClS3UjbV9C9DPzDqHXlLj8uXAO+5eS3T5bxlR\ngionSlKN23qt6Um4+yPAIwCFhYVa711E2oS7s3rLrgMj4OasrmR99R4A+nTrTFFeFlOLhlCUl83Y\n4/vQJSP95zlIZEKaC+SH0W/riQYpfKlJmVnAlcDbwGTgFXd3M5sF/NbM7iMa1JAPzCHq1RzUZqjz\namijOLT5XDjGs8AXgV+Z2QCiS3irgJXAv5tZ/1DuYqLBDyIiba6hwVn24Y4DyWf26kq27KwBYECv\nTMbnZXPtecMpysvihGN706lTrIs86S1hCSncE7oBeJHo3s3j7r7YzO4CStx9FvAY8GQYtFBJlGAI\n5WYSDYCoA65393qAWG2GQ94KFJvZ3cB7oW1C2YvNbAnREPJ/cfetoa0fECVOgLv2D3AQEUm02voG\nFm/YfmAAwtw1VWzbUwvA8X27cW7+AIrysijKy2L4gJ5E473aN3PXVah4FRYWupYwF5Ejsbe2ngXl\n25izeiuzwzNAu/eFZ4AG9DyQfIryshjcP/EPobYlM5vn7oUtlUv9YRciImloV00d766tOnD5rXRd\nNfvqGgAYPbA3k08fHCWg3CyO6dMtydGmBiUkEZFWsG13LSUfRPd/3lldyaL12w48A3TS8X248qxh\nFOVlc0Zuf/r1yEx2uClJCUlE5AhU7Khh7pp/DEB4f9N23CEzoxOnDOnLdROGU5SXzenD+tMrDZ4B\nSgX6LYmIxGFD9Z4DyWf26q2sqoieAerWpROnD+vPzReNoigvi4Ih/dLyGaBUoIQkItKEu7Nm6+4D\nAxDmrK6kvCp6Bqh3t86ckZvF5YXRPHAnHd+XzM7p/wxQKlBCEpEOr6HBWbF550cS0OYd0TNAWT0z\nKcrN4ppz8ijKy2L0wD5ktMNngFKBEpKIdDh19Q0s2bj9wCW4uWsqqd4dPQM0sE83zhqRTVFeFuPz\nshiR06tDPAOUCpSQRKTdq6mrZ2H5tnD/p5J5ayrZFZ4BGpbdg0+ceCzjh2czPi+Lwf27KwEliRKS\niLQ7u/fV8d7a6nD5bSvvra2mJjwDNOrYXvyf0wZRlJdNUW4WA/vqGaBUoYQkImlv+95a5q2pOjAC\nbmH5NuoanE4GY47vw1fOHHZgIbqsnnoGKFUpIYlI2tm6M3oGaP8AhCUbo2eAumQY4wb34+thEtLT\nh/WnT7fUW4hOYlNCEpGUt2nb3gOL0M1eXUnZ5p0AdO3cidOG9uebF+ZTlJfFqUP60z1TzwClKyUk\nEUkp7s7ayt0fWQdobeVuAHp17Uxhbn8uO20Q4/OyOHlQPz0D1I4oIYlI0lXv3sdbZVt4fVkFb67Y\nwqbtewHo36MLZ+RmceXHchmfl8Xogb3p3A4WopPYlJBEpM3VNzgL12/j9WUVvL58M6XrqmnwaCXU\nc/IHcNaIAYzPy2JkTq92uRCdxKaEJCJtYvOOvbyxfAuvL6/gzRUVVO+uxQzGDe7HDRfkM2FUDqcM\n7qseUAemhCQiCbGvroF311bx+vIKXl9WwZKN2wEY0KsrF4w+hgmjcjg3P0fDsOUAJSQRaTXrKnfz\n+vIK3lhewd9XbmVnTR2dOxmnD+vPdyaewIRROZw4sI8uw0lMSkgicsT21tbzzqqtUS9oecWBJRkG\n9evOpQXHM2FUDh8bkU1vPQskcVBCEpG4uTsrK3YdSECzV22lpq6Brp07cebwbL48fhgTRuUwIqen\n5oOTw6aEJCKHtGNvLX8r23rgUtz66mhdoBE5PaMEdEIO4/OytCidHDUlJBH5iIYGZ8nG7Qd6Qe9+\nUEVdg9Ora2c+NiKbb3x8BOfl5zAkq0eyQ5V2JqEJycwmAg8AGcCj7n5Pk/1dgV8DpwNbgSnuvibs\nmwZcA9QDN7r7i4dq08zygGIgC3gX+Kq77zOzq4CfAOvDYX/m7o+GOvXAwrB9rbtf2tq/A5F0ULlr\nH2+uqAi9oC1s2RktTjf2+D5ce95wJozK4bRh/emiIdmSQAlLSGaWATwIfAIoB+aa2Sx3X9Ko2DVA\nlbuPNLOpwL3AFDMbA0wFxgLHAy+b2ahQp7k27wWmu3uxmT0c2v55qDPD3W+IEeYedy9ozfMWSQd1\n9Q3ML68OD6ZWsGD9NtyjmRHOzc+JhmSPGsAxvbU0g7SdRPaQioAyd18FYGbFwCSgcUKaBNwZXj8N\n/MyiO6GTgGJ3rwFWm1lZaI9YbZrZUuAC4EuhzBOh3f0JSaTD27RtL2+Ey3Bvrqhg+946OhmcOrQ/\nN104igkn5HDyoL5anluSJpEJaRCwrtH7cmB8c2Xcvc7MtgHZYfs7TeoOCq9jtZkNVLt7XYzyAJ83\ns/OA5cDN7r6/jW5mVgLUAfe4+7NNT8LMrgWuBRg6dGhL5yySMmrq6ilZU3UgCb2/aQcAx/bpysST\nBjJh1DGcM3IAfXtoSLakhkQmpFh/ZnmcZZrbHusC9qHKA/wR+J2715jZdUS9pwvCvqHuvsHMhgOv\nmNlCd1/5kUbcHwEeASgsLGwav0hK+WDrrgMzI/x95Vb21NbTJcM4IzeLaZeMZsIJOZxwbG8NyZaU\nlMiEVA4MafR+MLChmTLlZtYZ6AtUtlA31vYtQD8z6xx6SQfKu/vWRuV/SXSvibBvf5lVZvYacCrw\nkYQkksp276uLHkwN94LWbI2WaRia1YMvFA5mwqgczhyeTc+uGlArqS+R/0rnAvlh9Nt6okEKX2pS\nZhZwJfA2MBl4xd3dzGYBvzWz+4gGNeQDc4h6Qge1Geq8GtooDm0+B2Bmx7n7xnC8S4GlYXt/YHfo\nOQ0AzgZ+nIDfg0ircXeWf7jzwGW4Oasr2VffQPcuGZw1Ipurz85jwqgccgf0THaoIoctYQkp3BO6\nAXiRaIj24+6+2MzuAkrcfRbwGPBkGLRQSZRgCOVmEg2AqAOud/d6gFhthkPeChSb2d3Ae6FtgBvN\n7NLQTiVwVdh+IvALM2sguhR4T5MRgCIpYdueWv4W1gp6fXnFgbWCRh3biys/NowJo46hMLe/HkyV\ntGfuui0Sr8LCQi8pKUl2GNLONTQ4izZsO5CA3ltXTX2D07tbZ87NH8CEUTmcNyqH4/p2T3aoInEx\ns3nuXthSOV1YFkkBFTtqeHNFNDXPGyu2ULlrHwDjBvflG+ePYMKoHAqG9NNaQdKuKSGJJEFtfQPv\nra3m9eWbeX15BYvW718rKJPzQw/onPwBDOjVNcmRirQdJSSRNrK+ek80GGFZBX8r28KOmjoyOhmn\nD+3Pv3wyWitozHFaK0g6LiUkkQTZW1vPnNWVByYpLdu8E4Dj+3bjM6ccF60VNHIAfbRWkAighCTS\natyd1Vv+sVbQO6u2sre2gczOnRifl8XUM4Zw/gk5jMjppQdTRWJQQhI5Cjtr6vh72RbeCDNlr6uM\n1goaPqAnU88YyoQTcjgzL5vumRqSLdISJSSRw+DuLN24I/SCNjPvgypq652emRl8bOQArj1vBBPy\ncxiarbWCRA6XEpJIC6p27eOtsi0HVkzdvCNaK+jE4/pwzTnRWkGnD+tPZmcNyRY5GkpIIk3UN/iB\ntYLeWFHB/HXVNDj07d7lIw+mHttHawWJtCYlJBFg8/a9BwYjvFW2herdtZhBwZB+3HhhPhNG5TBu\ncD+tFSSSQEpI0mGtq9zN/8xey+vLK1i6MXowNad3Vy468VgmjMrhnJED6N8zM8lRinQcSkjSIe2q\nqeOrj81mffUeCodlcevE0UwYlcOJx2mtIJFkUUKSDunuPy3lg8rd/O7rZ3Lm8OxkhyMixF6BVaRd\ne2nJh/xuzlr+73kjlIxEUogSknQoFTtquO33CxhzXB++9YlRyQ5HRBrRJTvpMNydW3+/gJ01dRRP\nLdBzQyIpRv8jpcP4n9lreeX9zUy7ZDT5x/ZOdjgi0oQSknQIKyt2cveflnDeqByuOCs32eGISAxK\nSNLu1dY3cPOMUrp1yeAnk8dpvSGRFKV7SNLu/edfV7CgfBsPf+U0TfcjksLUQ5J2bd4HlTz4ahlf\nOH0wE086LtnhiMghKCFJu7Wzpo6bZpQyqH937rh0bLLDEZEWJDQhmdlEM1tmZmVmdluM/V3NbEbY\nP9vMchvtmxa2LzOzT7bUppnlhTZWhDYzw/arzKzCzErDz9ca1bkylF9hZlcm6vcgyfH9WYtZX7WH\n6ZcX0Kurrk6LpLqEJSQzywAeBC4BxgBfNLMxTYpdA1S5+0hgOnBvqDsGmAqMBSYCD5lZRgtt3gtM\nd/d8oCq0vd8Mdy8IP4+GY2QBdwDjgSLgDjPr36q/BEmavyzayFPzyrn+4yMpzM1KdjgiEodE9pCK\ngDJ3X+Xu+4BiYFKTMpOAJ8Lrp4ELLZrZchJQ7O417r4aKAvtxWwz1LkgtEFo83MtxPdJ4CV3r3T3\nKuAlouQnae7D7Xu57Q8LGTe4LzdemJ/scEQkTolMSIOAdY3el4dtMcu4ex2wDcg+RN3mtmcD1aGN\nWMf6vJktMLOnzWzIYcSHmV1rZiVmVlJRUXHoM5aka2hwvv3UfPbW1jN9SgFdMnSbVCRdJPJ/a6yH\nPTzOMq21HeCPQK67jwNe5h89snjiw90fcfdCdy/MycmJUUVSya/fXsObK7Zw+6fHMCKnV7LDEZHD\nkMiEVA4MafR+MLChuTJm1hnoC1Qeom5z27cA/UIbHzmWu29195qw/ZfA6YcRn6SRFR/u4EcvvM8F\no4/hy+OHJjscETlMiUxIc4H8MPotk2iQwqwmZWYB+0e3TQZecXcP26eGUXh5QD4wp7k2Q51XQxuE\nNp8DMLPGD59cCiwNr18ELjaz/mEww8Vhm6ShfXUNfLO4lF5dO3Pv58dpkT2RNJSwsbDuXmdmNxB9\nyWcAj7v7YjO7Cyhx91nAY8CTZlZG1DOaGuouNrOZwBKgDrje3esBYrUZDnkrUGxmdwPvhbYBbjSz\nS0M7lcBV4RiVZvYDoiQHcJe7Vybo1yEJdt9Ly1mycTu/vKKQnN5dkx2OiBwBizoXEo/CwkIvKSlJ\ndhjSxOxVW5n6y3eYesYQfnTZuGSHIyJNmNk8dy9sqVxcl+zM7Gwz6xlef8XM7jOzYUcbpMjR2r63\nlm/NnM+wrB7c/ummj7mJSDqJ9x7Sz4HdZnYK8B3gA+DXCYtKJE53PLeYTdv3Mn1KAT01G4NIWos3\nIdWFgQOTgAfc/QFAK5xJUv1x/gaeeW89/3zBSE4dqkk2RNJdvH9S7jCzacBXgPPCFD5dEheWyKFt\n3LaH7z2zkIIh/bjh4yOTHY6ItIJ4e0hTgBrgGnffRDSjwU8SFpXIITQ0OLfMnE9dg3P/lAI6azYG\nkXYh7h4S0aW6ejMbBYwGfpe4sESa9/jfVvP3lVu557KTyR3QM9nhiEgrifdPyzeArmY2CPgrcDXw\nq0QFJdKc9zdt58d/WcYnxhzLlDOGtFxBRNJGvAnJ3H03cBnwX+7+f4iWhhBpM3tr67mpuJQ+3btw\nz2UnazYGkXYm7oRkZmcBXwb+FLZlJCYkkdh++r/LeH/TDn4yeRzZvTQbg0h7E29CugmYBjwTpvUZ\nTjR3nEib+FvZFn755mq+euYwPj76mGSHIyIJENegBnd/HXjdzHqbWS93XwXcmNjQRCLbdtdyy8z5\nDM/pyXc/dWKywxGRBIl36qCTzew9YBGwxMzmmZnuIUnCuTvfe3YhW3bW8MCUU+meqSvFIu1VvJfs\nfgF8y92HuftQ4BaitYVEEuq50g08v2AjN39iFCcP7pvscEQkgeJNSD3d/cA9I3d/DdADIJJQ5VW7\n+ddnF1E4rD/XTRiR7HBEJMHifTB2lZn9K/BkeP8VYHViQhKB+jAbgwPTpxSQ0UlDvEXau3h7SP8E\n5AB/AJ4Jr69OVFAiv3xzFbNXV3LnpWMZktUj2eGISBuId5RdFRpVJ21k0fpt/PR/l/Gpkwfy+dMG\nJTscEWkjh0xIZvZHoNklZd390laPSDq0vbX13DyjlP49Mvnh5zQbg0hH0lIP6T/aJAqR4J4X3mfF\n5p38+p+K6N8zM9nhiEgbOmRCCg/EirSJN5ZX8Ku/r+Hqs3M5b1ROssMRkTYW1z0kM1vIwZfutgEl\nwN3uvrW1A5OOpWrXPr791Hzyj+nFrRNHJzscEUmCeId9vwDUA78N76cCRpSUfgV8ttUjkw7D3Zn2\nh4VU7d7Hf199Bt26aDYGkY4o3mHfZ7v7NHdfGH6+B0xw93uB3OYqmdlEM1tmZmVmdluM/V3NbEbY\nP9vMchvtmxa2LzOzT7bUppnlhTZWhDYzmxxrspm5mRWG97lmtsfMSsPPw3H+LqSVPT2vnL8s3sS3\nLz6BscdrNgaRjirehNTLzMbvf2NmRUCv8LYuVgUzywAeBC4BxgBfNLMxTYpdA1S5+0hgOnBvqDuG\nqBc2FpgIPGRmGS20eS8w3d3zgarQ9v5YehMNW5/d5Pgr3b0g/FwX5+9CWtHarbu5c9Zixudl8bVz\nhyc7HBFJongT0teAR81stZmtAR4FvmZmPYEfNVOnCChz91Xuvg8oBiY1KTMJeCK8fhq40KJxvpOA\nYnevcffVQFloL2aboc4FoQ1Cm59rdJwfAD8G9sZ5vtIG6uob+NbMUjp1Mu7TbAwiHV5cCcnd57r7\nyUABUODu48K2Xe4+s5lqg4B1jd6Xh20xy7h7HdE9qexD1G1uezZQHdr4yLHM7FRgiLs/HyPGPDN7\nz8xeN7NzY52EmV1rZiVmVlJRUdHMqcqRePj1lZR8UMXdnzuJQf26JzscEUmyeJef6Gtm9wF/BV42\ns5+aWUsX+2P9udt0pF5zZVplu5l1IroUeEuM/RuBoe5+KvAt4Ldm1uegRtwfcfdCdy/MydFQ5Nay\noLya+19ewWdPOZ5JBZqNQUTiv2T3OLADuDz8bAf+u4U65cCQRu8HAxuaK2NmnYG+QOUh6ja3fQvQ\nL7TReHtv4CTgtXCp8UxglpkVhsuBWwHcfR6wEhjVwjlJK9i9r46bikvJ6d2VuyedlOxwRCRFxJuQ\nRrj7HeHezSp3/z7Q0h3ouUB+GP2WSTRIYVaTMrOAK8PrycAr7u5h+9QwCi8PyAfmNNdmqPNqaIPQ\n5nPuvs3dB7h7rrvnAu8Al7p7iZnlhEEShCXZ84FVcf4+5Cj8+5+XsmrLLn76hVPo26NLssMRkRQR\n73NIe8zsHHd/C8DMzgb2HKqCu9eZ2Q3Ai0AG8Li7Lzazu4ASd58FPAY8aWZlRD2jqaHuYjObCSwh\nGsV3vbvXh2Mf1GY45K1AsZndDbwX2j6U84C7zKyO6Bmr69y9Ms7fhxyhV9/fzG/eWcvXz83jYyMH\nJDscEUkhFnUuWihkVkA0cq0v0f2aSuAqd5+f2PBSS2FhoZeUlCQ7jLS1ZWcNE+9/gwG9uvLcDWfT\ntbMegBXpCMxsnrsXtlQu3uUnSoFT9t/0d/ftRxmfdDDuzm2/X8j2PXX85mvjlYxE5CAtLT/xrWa2\nA+Du9yUgJmmHZsxdx8tLP+T2T5/I6IEHDWYUEWmxh9S7TaKQdm31ll18/49LOHtkNv90dl6ywxGR\nFNXS8hPfb6tApH2qq2/g5hmldMkw/uMLp9BJszGISDPiHfZ9gJm9m4hApH362atllK6r5t8vO5nj\n+mo2BhFp3mEnJGLPiiBykHfXVvFfr5Rx2amD+My445MdjoikuCNJSH9q9Sik3dlVU8fNM0oZ2Kcb\nd04am+xwRCQNxPtg7AHufnsiApH25e4/LWFt5W6Kv34mfbppNgYRaVm8k6vuMLPtTX7WmdkzYdod\nkQP+d/EmfjdnHddNGMH44dnJDkdE0kS8PaT7iCYr/S3RPaSpwEBgGdHEq+cnIjhJP5t37OW2Pyxk\n7PF9uPkizVUrIvGL9x7SRHf/hbvvcPft7v4I8Cl3nwH0T2B8kkbcnVufXsCumjrun1JAZucjuUUp\nIh1VvN8YDWZ2uZl1Cj+XN9rX8mR40iH8ZvZaXl1WwXc/dSL5x+qZahE5PPEmpC8DXwU2Ax+G118x\ns+7ADQmKTdLIyoqd/PBPSzhvVA5XnDUs2eGISBqKd3LVVcBnm9n9VuuFI+motr6Bm4pL6d4lg59M\nHndgrkMRkcMR7yi7UWb2VzNbFN6PMzMN/xYAHnh5BQvXb+NHl53MsX26JTscEUlT8V6y+yUwDagF\ncPcFhMX0pGMrWVPJQ6+V8YXTBzPxpOOSHY6IpLF4E1IPd5/TZFtdawcj6WXH3lpunlnK4P49uONS\nzcYgIkcn3ueQtpjZCMKIOjObDGxMWFSSFr7/xyWsr9rDU9edRa+uhz3ph4jIR8T7LXI98Agw2szW\nA6uJRt5JB/XCwo08Pa+cf75gJKcPy0p2OCLSDsSbkNYD/w28CmQB24ErgbsSFJeksA+372XaMwsZ\nN7gvN16Yn+xwRKSdiDchPQdUA+8STSEkHVRDg/Ptp+ZTU9vA/VMK6JKh2RhEpHXE+20y2N2nuvuP\n3f2n+39aqmRmE81smZmVmdltMfZ3NbMZYf9sM8tttG9a2L7MzD7ZUptmlhfaWBHazGxyrMlm5mZW\n2NIxpHlPvL2GN1ds4XufPpHhOb2SHY6ItCPxJqS/m9nJh9OwmWUADwKXAGOAL5rZmCbFrgGq3H0k\nMB24N9QdQzSsfCwwEXjIzDJaaPNeYLq75wNVoe39sfQGbgRmN9oW8xiHc44dzfIPd3DPC+9zwehj\n+PL4ockOR0TamXgT0jnAvNCTWGBmC81sQQt1ioAyd1/l7vuAYmBSkzKTgCfC66eBCy16zH8SUOzu\nNe6+GigL7cVsM9S5ILRBaPNzjY7zA+DHwN4mx451DImhpq6em4pL6dW1M/d+XrMxiEjri/ce0iVH\n0PYgYF2j9+XA+ObKuHudmW0DssP2d5rUHRRex2ozG6h297qm5c3sVGCIuz9vZt9ucuzmjnGAmV0L\nXAswdGjH7RXc99JylmzczqNvHACJAAARz0lEQVRXFJLTu2uywxGRdijeuew+OIK2Y/0J3XRm8ObK\nNLc9Vo+u2fJm1onoUuBVRxgfYamNRwAKCws75Mzm76zayiNvrOKLRUO5aMyxyQ5HRNqpRD7NWA4M\nafR+MAeP0NtfptzMOgN9gcoW6sbavgXoZ2adQy9p//bewEnAa+ES00BglpldGmd8Hd62PbXcMnM+\nudk9+dfPnJjscESkHUvkmN25QH4Y/ZZJNIBgVpMys4ieZwKYDLzi7h62Tw2j8PKAfGBOc22GOq+G\nNghtPufu29x9gLvnunsu0SW6S9295BDHkEbueG4Rm7bvZfqUAnpkajYGEUmchH3DhHtCNwAvAhnA\n4+6+2MzuAkrcfRbwGPCkmZUR9YymhrqLzWwmsIRozrzr3b0eIFab4ZC3AsVmdjfwXmj7UPE1ewyJ\nzJq/gWdLN3DzRaMoGNIv2eGISDtnUedC4lFYWOglJSXJDqNNbKjew8T732DEMb146v+eRWc9ACsi\nR8jM5rl7YUvl9C0jB2locG6ZOZ+6Buf+KQVKRiLSJvRNIwd5/G+reXvVVu747BiGZfdMdjgi0kEo\nIclHLN24nR//ZRkXjzmWywuHtFxBRKSVKCHJAXtro9kY+nTvwo8uO1mzMYhIm9I4XjngP15cxrIP\nd/DfV59Bdi/NxiAibUs9JAHgb2VbePSt1Vxx1jA+fsIxyQ5HRDogJSShevc+bpk5nxE5PZl2iWZj\nEJHkUELq4Nyd7z27iC07a7h/yql0z9QKHCKSHEpIHdyzpev504KN3PyJUZw8uG+ywxGRDkwJqQMr\nr9rNvz27mDNy+3PdhBHJDkdEOjglpA6qvsH51sz5OHDf5QVkdNIQbxFJLg377qAeeWMVc1ZX8tMv\nnMKQrB7JDkdERD2kjmjR+m3c99IyPnXyQC477aBFckVEkkIJqYPZW1vPTTNKyeqZyQ8/p9kYRCR1\n6JJdB3PPC+9TtnknT15TRP+emckOR0TkAPWQOpDXl1fwq7+v4eqzczk3PyfZ4YiIfIQSUgdRuWsf\n335qPqOO7cWtE0cnOxwRkYPokl0H4O589w8Lqd69jyeuLqJbF83GICKpRz2kDuDpeeX8ZfEmvn3x\nCYw5vk+ywxERiUkJqZ1bu3U3d85azJnDs/jaucOTHY6ISLOUkNqxuvoGbp5ZSqdOxk81G4OIpDjd\nQ2rHHn59JfM+qOKBqQUM6tc92eGIiBxSQntIZjbRzJaZWZmZ3RZjf1czmxH2zzaz3Eb7poXty8zs\nky21aWZ5oY0Voc3MsP06M1toZqVm9paZjQnbc81sT9heamYPJ/J30dbmr6vm/pdXcOkpxzOpQLMx\niEjqS1hCMrMM4EHgEmAM8MX9yaCRa4Aqdx8JTAfuDXXHAFOBscBE4CEzy2ihzXuB6e6eD1SFtgF+\n6+4nu3sB8GPgvkbHX+nuBeHnutY8/2Tava+Om2eUktO7Kz+YdFKywxERiUsie0hFQJm7r3L3fUAx\nMKlJmUnAE+H108CFFs1lMwkodvcad18NlIX2YrYZ6lwQ2iC0+TkAd9/e6Hg9AW/l80w5//7npaze\nuoufXn4KfXt0SXY4IiJxSWRCGgSsa/S+PGyLWcbd64BtQPYh6ja3PRuoDm0cdCwzu97MVhL1kG5s\nVD/PzN4zs9fN7NxYJ2Fm15pZiZmVVFRUtHzWSfbK+x/ym3fW8vVzh/OxEQOSHY6ISNwSmZBiDelq\n2jtprkxrbY9euD/o7iOAW4Hbw+aNwFB3PxX4FvBbMzvoIR13f8TdC929MCcntafb2bKzhu88vYDR\nA3tzy8Wjkh2OiMhhSWRCKgeGNHo/GNjQXBkz6wz0BSoPUbe57VuAfqGN5o4F0SW+/Zfyatx9a3g9\nD1gJpO23uLtz2+8Xsn1vHfdPLaBrZ83GICLpJZEJaS6QH0a/ZRINUpjVpMws4MrwejLwirt72D41\njMLLA/KBOc21Geq8GtogtPkcgJnlNzrep4EVYXtOGCSBmQ0Px1jVamffxornruPlpR9y68TRjB6o\n2RhEJP0k7Dkkd68zsxuAF4EM4HF3X2xmdwEl7j4LeAx40szKiHpGU0PdxWY2E1gC1AHXu3s9QKw2\nwyFvBYrN7G7gvdA2wA1mdhFQSzT6bn8CPA+4y8zqgHrgOnevTNTvI5FWb9nFXX9cwtkjs7n6Y7nJ\nDkdE5IhY1LmQeBQWFnpJSUmyw/iI2voGJj/8Nmu27OIvN53LcX31AKyIpBYzm+fuhS2V00wNae5n\nr5Qxf101D37pNCUjEUlrmssujb27toqfvVrGZacO4tPjjkt2OCIiR0UJKU3tqolmYxjYpxt3Thqb\n7HBERI6aLtmlqR88v4S1lbuZce1Z9Omm2RhEJP2ph5SGXly8ieK567huwgiK8rKSHY6ISKtQQkoz\nm3fsZdofFjL2+D7cfFHaPscrInIQJaQ04u585+kF7Kqp44GpBWR21scnIu2HvtHSyG/e+YDXllXw\n3U+dyMhjeic7HBGRVqWElCbKNu/kh39eyoRROVxx1rBkhyMi0uqUkNLAvroGbprxHt27ZPCTyeOI\nln8SEWlfNOw7DTzw1+UsWr+dh79yOsf06ZbscEREEkI9pBQ3d00lP39tJZcXDmbiSQOTHY6ISMIo\nIaWwHXtruXlGKYP79+DfPqvZGESkfdMluxR256wlbKjew1PXfYxeXfVRiUj7ph5Sivrzwo38/t1y\nbvj4SE4f1j/Z4YiIJJwSUgratG0v331mIacM7ss/X5jfcgURkXZACSnFNDQ4//L0fGpqG5g+pYAu\nGfqIRKRj0Lddinni7TW8uWILt3/mRIbn9Ep2OCIibUYJKYUs/3AHP3rhfS4cfQxfKhqa7HBERNqU\nElKKqKmr55vFpfTu2pl7Pq/ZGESk49FY4hRx30vLWbpxO49eUUhO767JDkdEpM2ph5QC3l65lUfe\nWMWXxg/lojHHJjscEZGkSGhCMrOJZrbMzMrM7LYY+7ua2Yywf7aZ5TbaNy1sX2Zmn2ypTTPLC22s\nCG1mhu3XmdlCMys1s7fMbExLx2hL2/bUcsvMUnKze3L7p09MRggiIikhYQnJzDKAB4FLgDHAFxsn\ng+AaoMrdRwLTgXtD3THAVGAsMBF4yMwyWmjzXmC6u+cDVaFtgN+6+8nuXgD8GLjvUMdo5V9Di+54\nbhEf7qhh+pQCemTqCqqIdFyJ7CEVAWXuvsrd9wHFwKQmZSYBT4TXTwMXWnQ3fxJQ7O417r4aKAvt\nxWwz1LkgtEFo83MA7r690fF6At7o2LGO0WaeK13Ps6Ub+OaF+RQM6deWhxYRSTmJTEiDgHWN3peH\nbTHLuHsdsA3IPkTd5rZnA9WhjYOOZWbXm9lKoh7SjYcRH2Z2rZmVmFlJRUVFC6ccvw3Ve7j92UWc\nOrQf3zh/RKu1KyKSrhKZkGKNW/Y4y7TW9uiF+4PuPgK4Fbj9MOLD3R9x90J3L8zJyYlR5fA1NDi3\nzJxPfYNz/5QCOms2BhGRhCakcmBIo/eDgQ3NlTGzzkBfoPIQdZvbvgXoF9po7lgQXeL73GHElxCP\nvbWat1dt5c7PjmVYds+2OKSISMpLZEKaC+SH0W+ZRAMIZjUpMwu4MryeDLzi7h62Tw2j8PKAfGBO\nc22GOq+GNghtPgdgZo1nJ/00sKLRsWMdI6GWbtzOT15cxsVjjuULhYMTfTgRkbSRsGFd7l5nZjcA\nLwIZwOPuvtjM7gJK3H0W8BjwpJmVEfWMpoa6i81sJrAEqAOud/d6gFhthkPeChSb2d3Ae6FtgBvM\n7CKglmj03ZUtHSNR9tbWc1NxKX17dNFsDCIiTVjUuZB4FBYWeklJyRHX/8HzS3jsrdX86uozOP+E\nY1oxMhGR1GVm89y9sKVyupveRt5asYXH3lrNFWcNUzISEYlBCakNVO/ex7efms+InJ5Mu0SzMYiI\nxKKpAdpAXYNz0qC+3HRRPt0z23wyCBGRtKCE1AYG9OrKo1e2ePlURKRD0yU7ERFJCUpIIiKSEpSQ\nREQkJSghiYhISlBCEhGRlKCEJCIiKUEJSUREUoISkoiIpARNrnoYzKwC+OAomhhAtHZTumsv5wE6\nl1TVXs6lvZwHHN25DHP3Flc4VUJqQ2ZWEs+Mt6muvZwH6FxSVXs5l/ZyHtA256JLdiIikhKUkERE\nJCUoIbWtR5IdQCtpL+cBOpdU1V7Opb2cB7TBuegekoiIpAT1kEREJCUoIYmISEpQQmplZjbRzJaZ\nWZmZ3RZjf1czmxH2zzaz3LaPMj5xnMtVZlZhZqXh52vJiLMlZva4mW02s0XN7Dcz+89wngvM7LS2\njjFecZzL+Wa2rdFn8m9tHWM8zGyImb1qZkvNbLGZfTNGmbT4XOI8l3T5XLqZ2Rwzmx/O5fsxyiTu\nO8zd9dNKP0AGsBIYDmQC84ExTcp8A3g4vJ4KzEh23EdxLlcBP0t2rHGcy3nAacCiZvZ/CngBMOBM\nYHayYz6KczkfeD7ZccZxHscBp4XXvYHlMf59pcXnEue5pMvnYkCv8LoLMBs4s0mZhH2HqYfUuoqA\nMndf5e77gGJgUpMyk4AnwuungQvNzNowxnjFcy5pwd3fACoPUWQS8GuPvAP0M7Pj2ia6wxPHuaQF\nd9/o7u+G1zuApcCgJsXS4nOJ81zSQvhd7wxvu4SfpiPfEvYdpoTUugYB6xq9L+fgf5gHyrh7HbAN\nyG6T6A5PPOcC8PlwOeVpMxvSNqG1unjPNV2cFS65vGBmY5MdTEvCJZ9Tif4abyztPpdDnAukyedi\nZhlmVgpsBl5y92Y/l9b+DlNCal2x/kpo+tdFPGVSQTxx/hHIdfdxwMv846+mdJMun0k83iWaN+wU\n4L+AZ5MczyGZWS/g98BN7r696e4YVVL2c2nhXNLmc3H3encvAAYDRWZ2UpMiCftclJBaVznQuJcw\nGNjQXBkz6wz0JTUvwbR4Lu6+1d1rwttfAqe3UWytLZ7PLS24+/b9l1zc/c9AFzMbkOSwYjKzLkRf\n4P/j7n+IUSRtPpeWziWdPpf93L0aeA2Y2GRXwr7DlJBa11wg38zyzCyT6IbfrCZlZgFXhteTgVc8\n3B1MMS2eS5Pr+ZcSXTtPR7OAK8KorjOBbe6+MdlBHQkzG7j/er6ZFRH9H9+a3KgOFmJ8DFjq7vc1\nUywtPpd4ziWNPpccM+sXXncHLgLeb1IsYd9hnVujEYm4e52Z3QC8SDRK7XF3X2xmdwEl7j6L6B/u\nk2ZWRvRXxdTkRdy8OM/lRjO7FKgjOperkhbwIZjZ74hGOQ0ws3LgDqKbtbj7w8CfiUZ0lQG7gauT\nE2nL4jiXycD/M7M6YA8wNUX/4Dkb+CqwMNyvAPguMBTS7nOJ51zS5XM5DnjCzDKIkuZMd3++rb7D\nNHWQiIikBF2yExGRlKCEJCIiKUEJSUREUoISkoiIpAQlJBERSQlKSCIpxsx2tlzqkPWfNrPh4XUv\nM/uFma0Msze/YWbjzSwzvNajH5IylJBE2pEwR1qGu68Kmx4lelYk393HEj0rNiBMmPtXYEpSAhWJ\nQQlJJEWFGQp+YmaLzGyhmU0J2zuZ2UOhx/O8mf3ZzCaHal8GngvlRgDjgdvdvQEgzN7+p1D22VBe\nJCWouy6Sui4DCoBTgAHAXDN7g2hmgFzgZOAYoimbHg91zgZ+F16PBUrdvb6Z9hcBZyQkcpEjoB6S\nSOo6B/hdmH35Q+B1ogRyDvCUuze4+ybg1UZ1jgMq4mk8JKp9Zta7leMWOSJKSCKpq7lFzw61GNoe\noFt4vRg4xcwO9f+8K7D3CGITaXVKSCKp6w1gSlgwLYdo+fI5wFtECyN2MrNjiSZb3W8pMBLA3VcC\nJcD3G800nW9mk8LrbKDC3Wvb6oREDkUJSSR1PQMsAOYDrwDfCZfofk+0Js0i4BdEq5NuC3X+xEcT\n1NeAgUCZmS0kWrdq/5pCHyeaUVskJWi2b5E0ZGa93H1n6OXMAc52901hDZtXw/vmBjPsb+MPwDR3\nX9YGIYu0SKPsRNLT82EhtUzgB6HnhLvvMbM7gEHA2uYqh0UXn1UyklSiHpKIiKQE3UMSEZGUoIQk\nIiIpQQlJRERSghKSiIikBCUkERFJCf8fD7jDBVd6pq8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1167d588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L1.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这个score似乎和GridSearchCV得到的Score不一样? 拐点位置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  3.22029868e-01,   7.76912017e-01,  -3.33139991e+00,\n",
       "         -3.12964750e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.01512939e-01,   6.60603417e-03,  -2.67754933e-03,\n",
       "         -1.02262536e-01,   1.08716759e-01,   2.76993494e-01,\n",
       "         -2.35923276e-03,  -1.39161208e-01,  -1.36892323e-01,\n",
       "          4.87794711e-02,  -1.01019045e-01,  -3.38757174e-02,\n",
       "          5.36622599e-02,   2.49249870e-01],\n",
       "       [  1.68102782e-01,   5.27100123e-01,  -1.51378215e+00,\n",
       "         -2.16337274e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "         -2.27348935e-02,  -3.65671903e-03,  -1.52188868e-02,\n",
       "          7.30446850e-02,   9.59670802e-02,   8.61438665e-02,\n",
       "         -2.10823282e-02,  -1.69029999e-01,   4.55478923e-02,\n",
       "          3.15573442e-02,  -5.03338570e-02,   3.14548975e-02,\n",
       "          7.28590505e-02,   2.63807675e-01],\n",
       "       [ -2.84969316e-01,  -7.28445752e-01,   2.32456951e+00,\n",
       "          2.78067577e-01,   0.00000000e+00,  -1.24617689e-02,\n",
       "          5.09871022e-02,  -9.09144623e-04,   2.61427512e-02,\n",
       "         -2.47811132e-02,  -1.23155551e-01,  -1.70680492e-01,\n",
       "          2.22387162e-02,   1.97277471e-01,   0.00000000e+00,\n",
       "         -4.17161504e-02,   8.19858094e-02,  -6.60421719e-03,\n",
       "         -9.43089980e-02,  -3.11433524e-01]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.24279172, -0.24280583, -0.24279136, -0.24278846],\n",
       "        [-0.23781776, -0.23781531, -0.23782253, -0.23782439],\n",
       "        [-0.23874114, -0.23876975, -0.2388056 , -0.23881315],\n",
       "        [-0.23187505, -0.23181349, -0.2318072 , -0.23180804],\n",
       "        [-0.24163472, -0.24179882, -0.24195338, -0.24198539]]),\n",
       " 1: array([[-0.50495348, -0.50494615, -0.50494726, -0.50494766],\n",
       "        [-0.49692669, -0.49703541, -0.49709888, -0.497108  ],\n",
       "        [-0.496337  , -0.49630416, -0.49629026, -0.49628833],\n",
       "        [-0.50066779, -0.50071502, -0.50079052, -0.50080582],\n",
       "        [-0.50126432, -0.50126201, -0.50126672, -0.50126769]]),\n",
       " 2: array([[-0.54998561, -0.54994629, -0.5499336 , -0.54993222],\n",
       "        [-0.53597045, -0.53611186, -0.53619522, -0.53620648],\n",
       "        [-0.53738138, -0.53737004, -0.53736891, -0.5373688 ],\n",
       "        [-0.53945439, -0.53948147, -0.53955885, -0.53956655],\n",
       "        [-0.54821878, -0.54836571, -0.54844282, -0.54845387]])}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaYAAAERCAYAAAAwvgPOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8FeW9x/HPj33fAyKLLIIKiCwh\ngNu1rSLaVtzXlrAoinrb3ta2tvaWq7a3tt7aSq0oKpuKiFqVupQi1VoLSAIiIC4EEAgghC3sS5Lf\n/eMMeoznkEPIyZyTfN+v13ll5plnnvk9HswvM/PMPObuiIiIpIoaYQcgIiISTYlJRERSihKTiIik\nFCUmERFJKUpMIiKSUpSYREQkpSgxJZGZ3WFmbmatYmzrY2bzzewDM1tqZtdEbfuXmS0JPhvN7KUK\niOXXZrbezPYcb1siIsmkxHSczOw8M5sSo7wDcAGwLs6u+4Dh7t4TGAr80cyaAbj7Oe7ex937APOB\nv1RAqH8FsiqgHRGRpFJiSp4/AD8BYj7B7O6fuPvKYHkjsAXIiK5jZo2BrwMvBesNzWySmeWY2Xtm\nNizRYNx9gbtvKl9XREQqjxJTEpjZJcAGd38/wfpZQB1gValNlwFz3X1XsH4X8A93HwB8DbjfzBpW\nUNgiIimhVtgBpCszexeoCzQCWpjZkmDTOODnwJAE22kLPAlku3tJqc3XAY9HrQ8BLjGzO4L1ekDH\n4L7R63EOMSQ4IxMRSQumd+UdHzM7Dxjh7iOC9dOBuUTuIQG0BzYCWe7+Wal9mwBvAb9x9+dKbWsJ\nfAK0c/cDQdki4Hp3//g44t3j7o3Ku7+ISLLpUl4Fc/dl7t7a3Tu5eycgH+gXIynVAV4EppVOSoGr\ngFeOJKXAbOA/zcyCNvompRMiIiFSYqpEZpZpZkcuzV0NnAuMiBoa3ieq+rXAM6WauBeoDSw1s+XB\neqLH/p2Z5QMNzCzfzP6n3B0REUkiXcoTEZGUojMmERFJKRqVVw6tWrXyTp06hR2GiEhaWbRo0VZ3\nzyirnhJTOXTq1Inc3NywwxARSStmtjaRerqUJyIiKUWJSUREUooSk4iIpBQlJhERSSlKTCIiklKU\nmEREJKUoMYmISErRc0wiIhJXSYmzsXA/qwr2smrLHurVrsn1Azsm9ZihJiYzGwo8CNQEHnf3+0pt\nrwtMA/oD24Br3P3TYNvPgNFAMfA9d599tDbNrDMwA2gBLAa+6+6HzOxc4I9Ab+Bad38+qZ0WEUlB\n+w8Vs3rrns8T0KqCyPKarXs4cPiLqeL6dWxWdROTmdUE/gxcQGRqiBwzm+XuK6KqjQZ2uPvJZnYt\n8FvgGjPrQeTt2z2BE4E3zKx7sE+8Nn8L/MHdZ5jZI0HbE4B1wAjgDkREqjB3p2DPQVZt2Rskni8S\n0Yad+z+vZwYdmjega0ZDzurakq6tG9E1oxFdMxrSomGdpMcZ5hlTFpDn7qsBzGwGMAyITkzDgP8J\nlp8HHgrmIhoGzHD3g8AaM8sL2iNWm2b2IfB14PqgztSg3QlRZ2ClZ48VEUlLh4tLWLtt3xfJJyoR\n7T5Q9Hm9+rVr0rV1QzI7NeeajA6R5NO6IZ1aNqRe7ZqhxR9mYmoHrI9azwcGxqvj7kVmVgi0DMoX\nlNq3XbAcq82WwE53L4pRPyFmNgYYA9CxY3JPY0VEElG47zCrtu4JLr19kXzWbdtHUckXUxqd0KQe\nXTIacmmfdnTNaPj5GdAJTepRo4aF2IPYwkxMsf5rlJ4cKl6deOWxRhkerX7C3H0iMBEgMzNTk1iJ\nSKUoKXE27Nz/xWW3gi8S0dY9Bz+vV7um0blVQ05p05iLe7Wla+uGdM1oROdWDWlcr3aIPTh2YSam\nfKBD1Hp7YGOcOvlmVgtoCmwvY99Y5VuBZmZWKzhrinUsEZHQxBt8sLpgDweLvrjT0KxBbU7OaMQ3\nTm39efLpmtGI9s3rU6tm1XgCKMzElAN0C0bLbSAymOH6UnVmAdnAfOBK4B/u7mY2C5huZg8QGfzQ\nDVhI5MzoK20G+7wZtDEjaPPlZHdQRCSau1Ow+yB5BV9OQKsL9n5p8EENgw4tGtA1oxFnn9wyuPcT\nSUCVMfggbKElpuCe0e3AbCJDuye5+wdmdg+Q6+6zgCeAJ4PBDduJJBqCejOJDJQoAm5z92KAWG0G\nh/wpMMPMfgW8F7SNmQ0AXgSaA982s7vdvWcl/CcQkSrqUFEJ67bvJa/U6LfVW/aw++AXgw8a1KlJ\n14xGDOjUnGszOnyefE5q2SDUwQdhM3fdLjlWmZmZrokCRaRw3+Hg7OeL0W+rC/awdvs+iksNPoi+\n7HZk9NsJTeoRGWhcPZjZInfPLKue3vwgInIUxSXOxp37IwkoavTb6oI9bN1z6PN6dWrWoFOrBpxy\nQmMuPv2LwQddMhrRqK5+1R4L/dcSEQH2HSpi9edDrr8Y/bZm6944gw/aVNnBB2FTYhKRaqM8gw/O\n6daq2g0+CJsSk4hUOYeKSli7be+XXrlzZHlP1OCDhnVq0rW1Bh+kGiUmEUlbO/cd+sord1YV7GVd\nqcEHbZvWo2tGI67o1y7qvW+NaNOkbrUafJAulJhEJKUVlzgbduyPSjxfJKJte788+KBzq4ac1rYx\n3+rd9vPk0zmjoQYfpBl9WyKScgp2H+Q3r3/Iio27WL11L4eiBh+0aFiHrhkNuaBHm8+HXUcGHzSg\nZgq+902OnRKTiKSUT7fuZfikhWzZfYCzurbi3O4ZkRePBkOvNfig6lNiEpGUsXxDISMmL6S4xHnm\npkH07dg87JAkBEpMIpIS3lm5lZufzKVZgzpMG51F14xGYYckIVFiEpHQ/fX9jfxw5hK6ZjRi6qgs\n2jSpF3ZIEiIlJhEJ1ZR/r+HuV1Yw4KQWPJadSdP66TV3kFQ8JSYRCYW7c//sj3n4rVUM6dGG8df1\n1UOtAigxiUgIiopL+PmLy5iZm891WR351aW9NNRbPqfEJCKVav+hYm6fvpi5H23h+9/oxg/O76a3\nL8iXKDGJSKXZue8Qo6fmsnjdDu69tBffHXRS2CFJClJiEpFKsXHnfrInLWTttn38+fp+XHx627BD\nkhQV6uQhZjbUzD42szwzuzPG9rpm9myw/V0z6xS17WdB+cdmdmFZbZpZ56CNlUGbdco6hohUjJWb\nd3PFhHl8VniAqaOylJTkqEJLTGZWE/gzcBHQA7jOzHqUqjYa2OHuJwN/AH4b7NsDuBboCQwFHjaz\nmmW0+VvgD+7eDdgRtB33GCJSMRat3c6Vj8ynqMR59ubBDO7aMuyQJMWFecaUBeS5+2p3PwTMAIaV\nqjMMmBosPw98wyJ3SYcBM9z9oLuvAfKC9mK2Gezz9aANgjYvLeMYInKc5n64mRsef5fmDWrzl7Fn\n0uPEJmGHJGkgzMTUDlgftZ4flMWs4+5FQCHQ8ij7xitvCewM2ih9rHjH+BIzG2NmuWaWW1BQcEwd\nFamOZuauZ8yTi+jepjHPjz2TDi0ahB2SpIkwE1OssxJPsE5FlScaB+4+0d0z3T0zIyMjxi4iApEH\nZx9+K4+fPL+UM7u2ZPpNg2jVqG7YYUkaCXNUXj7QIWq9PbAxTp18M6sFNAW2l7FvrPKtQDMzqxWc\nFUXXj3cMETlGJSXOPa+sYMq8TxnW50Tuv/IM6tQKdYyVpKEw/8XkAN2C0XJ1iAxmmFWqziwgO1i+\nEviHu3tQfm0woq4z0A1YGK/NYJ83gzYI2ny5jGOIyDE4WFTM959dwpR5nzL67M784eo+SkpSLqGd\nMbl7kZndDswGagKT3P0DM7sHyHX3WcATwJNmlkfkLObaYN8PzGwmsAIoAm5z92KAWG0Gh/wpMMPM\nfgW8F7RNvGOISOL2HCzilicX8U7eVu686FRuPreL3uYg5WY6OTh2mZmZnpubG3YYIilh656DjJyc\nw4pNu/jtFb25sn/7sEOSFGVmi9w9s6x6evODiJTbum37GD7pXT7bdYDHhvfn66e2CTskqQKUmESk\nXCLToOdQVFLC9JsG0U/ToEsFUWISkWM2L28rY55cRJN6tZgxZjAnt24cdkhShSgxicgxeXXpJv7r\n2SV0atWAqaOyaNu0ftghSRWjxCQiCZs2/1PGzfqA/h2b80T2AJo20DToUvGUmESkTO7OA3M+4U//\nyOP809rw0PWaBl2SR4lJRI6qqLiEX7y0nBk567l2QAd+dWkvatXUg7OSPEpMIhLXgcPF3D79Pd74\ncDP/+fWT+eEF3fXgrCSdEpOIxFS47zA3Tsshd+0O7r6kJ9lndgo7JKkmlJhE5Cs+KzxA9qSFrNm6\nlz9d15dv9T4x7JCkGlFiEpEvyduyh+xJCyncf5gpIwdw5smtwg5JqhklJhH53OJ1Oxg1JYdaNWow\nY8wgerVrGnZIUg0pMYkIAG9+tIWxTy+iTZN6TBuVxUktG4YdklRTSkwiwguL8vnJC0s5rW1jJo/I\nIqOxZpyV8CgxiVRj7s7Et1fzm9c/4qyTW/LodzNpVFe/FiRc+hcoUk2VlDi/fu1DnnhnDd/q3Zbf\nX30GdWvpbQ4SvlAe3zazFmY2x8xWBj9jvi/fzLKDOivNLDuqvL+ZLTOzPDMbb8ETf/HatYjxQf2l\nZtYvqq2/mdlOM3sl2f0WSRWHikr4r5lLeOKdNYw4sxPjr+2rpCQpI6z3itwJzHX3bsDcYP1LzKwF\nMA4YCGQB46IS2ARgDNAt+Awto92LouqOCfY/4n7guxXWM5EUt+dgEaOn5vDyko38+MJTGPftHtSo\nobc5SOoIKzENA6YGy1OBS2PUuRCY4+7b3X0HMAcYamZtgSbuPt8j88JPi9o/XrvDgGkesQBoFrSD\nu88Fdlds90RS07Y9B7n+sQXMW7WN313Rm9u+drJeMSQpJ6x7TG3cfROAu28ys9Yx6rQD1ket5wdl\n7YLl0uVHazdeW5uOtyMi6WL99n0Mn7SQjTv38+h3+nN+D02DLqkpaYnJzN4AToix6a5Em4hR5kcp\nL09bCTOzMUQuA9KxY8dj2VUkdCs27iJ78kIOFZUw/aaB9D+pRdghicSVtMTk7ufH22Zmm82sbXBW\n0xbYEqNaPnBe1Hp74K2gvH2p8o3Bcrx284EOcfZJiLtPBCYCZGZmHlNSEwnT/FXbGDMtl0b1ajH9\nlsF0a6Np0CW1hXWPaRZwZJRdNvByjDqzgSFm1jwY9DAEmB1cqtttZoOC0XjDo/aP1+4sYHgwOm8Q\nUHjkkp9IVfb6sk1kT1pIm6b1eGHsmUpKkhbCSkz3AReY2UrggmAdM8s0s8cB3H07cC+QE3zuCcoA\nxgKPA3nAKuD1o7ULvAasDuo/Btx6JBAz+xfwHPANM8s3swuT0mORSvbUgrXcOn0xvdo14flbBnNi\ns/phhySSEIsMbJNjkZmZ6bm5uWGHIRKTu/PHN1by4NyVfOPU1jx0fT/q19EzShI+M1vk7pll1dOb\nH0SqkOIS5xcvLeeZheu4qn97fnP56ZoGXdKOEpNIFXHgcDHfn/Eesz/YzK3ndeXHF56iZ5QkLSkx\niVQBhfsPc9PUXBZ+up1ffqsHo87uHHZIIuWmxCSS5jbvikyDvqpgD+Ov68slZ2gadElvSkwiaWxV\nwR6GP7GQnfsOMXlEFmd30zTokv6UmETS1JL1Oxk5eSE1zJgxZjCnt9c06FI1KDGJpKG3Pt7C2KcW\n06pxHaaNGkjnVpoGXaoOJSaRNPPie/n8+LmldG/TmCmjBtC6cb2wQxKpUEpMImnksbdX8+vXPmRw\nl5ZMHN6fxvVqhx2SSIVTYhJJAyUlzn1/+4iJb6/mm6e35YFrNA26VF1KTCIp7nBxCT95fikvvreB\n4YNPYty3e1JTM85KFabEJJLC9h4s4tanF/PPTwr40QXduf3rmnFWqj4lJpEUtX3vIUZOyWFZ/k7u\nu/x0rs3SBJVSPST0dkczO8vMGgbL3zGzB8zspOSGJlJ95e/Yx5WPzOOjTbt45Dv9lZSkWkn0tcMT\ngH1mdgbwE2AtMC1pUYlUYx99tosrJsxj6+6DPHXjQIb0PCHskEQqVaKJqcgjEzcNAx509wcBTYUp\nUsHeXb2Nqx6ZD8Bzt5zJgE4tQo5IpPIleo9pt5n9DPgOcK6Z1QT0AIVIBfrb8s/43oz3aN+8PtNG\nZdG+eYOwQxIJRaJnTNcAB4HR7v4Z0A64v7wHNbMWZjbHzFYGP5vHqZcd1FlpZtlR5f3NbJmZ5ZnZ\neAuGKcVr1yLGB/WXmlm/oLyPmc03sw+C8mvK2yeR4zH93XXc+vQierRtwvO3nKmkJNVaoolpN5FL\neP8ys+5AH+CZ4zjuncBcd+8GzA3Wv8TMWgDjgIFAFjAuKoFNAMYA3YLP0DLavSiq7phgf4B9wHB3\n7xm08Ucza3Yc/RI5Ju7Og2+s5OcvLuM/umcw/aaBtGhYJ+ywREKVaGJ6G6hrZu2I/MIfCUw5juMO\nA6YGy1OBS2PUuRCY4+7b3X0HMAcYamZtgSbuPj+47zUtav947Q4DpnnEAqCZmbV190/cfSWAu28E\ntgAZx9EvkYQVlzj//fJy/vDGJ1zerx0Th2fSoI6e4BBJNDGZu+8DLgf+5O6XAT2P47ht3H0TQPCz\ndYw67YD1Uev5QVm7YLl0+dHajdfW58wsC6gDrCpHf0SOyYHDxdw+fTFPLVjHzf/Rhd9fdQa1ayb6\nv6NI1Zbon2dmZoOBG4DRQdlRX9RlZm8Asca53pXoMWOU+VHKy9NWZGPkLOxJINvdS2I2YDaGyGVA\nOnbUMyVSfrsORKZBf3fNdn7xzdO48ZwuYYckklISTUw/AH4GvOjuH5hZF+DNo+3g7ufH22Zmm4NL\naZuCpLAlRrV84Lyo9fbAW0F5+1LlG4PleO3mAx1i7WNmTYBXgV8El/ni9WciMBEgMzOzrEQoEtOW\nXQfInpzDys27+eM1fbi0b7uydxKpZhK6duDu/3T3S4CHzayRu6929+8dx3FnAUdG2WUDL8eoMxsY\nYmbNg0EPQ4DZwSW63WY2KBiNNzxq/3jtzgKGB6PzBgGFQfKqA7xI5P7Tc8fRH5Eyrdm6l8snzGPt\ntr1MGjFASUkkjkRfSXS6mb0HLAdWmNkiMzuee0z3AReY2UrggmAdM8s0s8cB3H07cC+QE3zuCcoA\nxgKPA3lE7gm9frR2gdeA1UH9x4Bbg/KrgXOBEWa2JPj0OY5+icS0NH8nV06Yx75DxTxz0yDO7a4x\nNiLxWGRgWxmVzOYBd7n7m8H6ecD/uvuZyQ0vNWVmZnpubm7YYUiaePuTAm55ahEtGtZh2qgsumQ0\nCjskkVCY2SJ3zyyrXqL3mBoeSUoA7v7WkZe6ikh8Ly/ZwI9mvs/JrRsxbVQWrZtoGnSRsiSamFab\n2X8TGbkGkVcTrUlOSCJVwxPvrOHeV1YwsHMLHsvOpImmQRdJSKIPTowi8uDpX4gMFsgg8pCtiJTi\n7tz3+kfc+8oKLup1AlNHZSkpiRyDhM6YgjcvHM8oPJFq4XBxCXe+sIwXFudzw8CO3DOsl6ZBFzlG\nR01MZvZXjvLwajCEXESAfYeKuO3pxbz5cQH/dX53vvcNTYMuUh5lnTH9X6VEIZLmdgTToC/N38mv\nL+vFDQM1wbNIeR01Mbn7PysrEJF0tWHnfoY/8S7rd+zn4Rv6M7SXZpwVOR4J3WMys2V89ZJeIZAL\n/Mrdt1V0YCLp4OPPdpM9aSF7DxXx5KgsBnZpGXZIImkv0eHirwPFwPRg/VoiL0YtJDL9xbcrPDKR\nFJfz6XZGT8mhXu2azLx5MKe1bRJ2SCJVQqKJ6Sx3PytqfZmZ/dvdzzKz7yQjMJFUNmfFZm6fvph2\nzeozdVQWHVpoxlmRipLoc0yNzGzgkZVg7qIj71UpqvCoRFLYjIXruPnJXE5t24TnbhmspCRSwRI9\nY7oRmGRmjYhcwtsFjA5eS/SbZAUnkkrcnT+/mcf//f0Tzu2ewYQb+tGwrmacFaloiT5gmwOcbmZN\nibz4dWfU5plJiUwkhRSXOPf89QOmzl/LZX3b8bsre2vGWZEkSXRUXlNgHJEpIjCzfxKZhqIwibGJ\npISDRcX88Nn3eXXZJm46pzM/u+g0auhtDiJJk+iffJOA3UTmL7qayKW8yckKSiRV7D5wmJGTc3h1\n2SZ+fvGp3PXNHkpKIkmW6AXyru5+RdT63Wa2JBkBiaSKLbsPMGJSDp9s3s0DV5/B5f3ahx2SSLWQ\n6BnTfjM7+8iKmZ0F7E9OSCLh+3TrXq6cMJ81W/fyWHamkpJIJUo0MY0F/mxmn5rZWuAh4JbyHtTM\nWpjZHDNbGfxsHqdedlBnpZllR5X3N7NlZpZnZuMteFNmvHYtYnxQf6mZ9QvKTwqmiV9iZh+YWbn7\nJFXH8g2FXPnIPHYfOMz0mwbytVNahx2SSLWSUGJy9yXufgbQGzjd3fu6+/vHcdw7gbnu3g2YG6x/\niZm1IDLgYiCQBYyLSmATgDFAt+AztIx2L4qqOybYH2ATcKa79wmOc6eZnXgc/ZI0987KrVzz6Hzq\n1qrJ82PPpG/HmH8ziUgSlTXtxQ/jlAPg7g+U87jDgPOC5anAW8BPS9W5EJjj7tuDY84BhprZW0AT\nd58flE8DLiXy2qR47Q4Dprm7AwvMrJmZtXX3TVHHq0viZ5BSBc16fyM/mrmELq0aMXVUFic01TTo\nImEoa/BD4yQdt82RpODum8ws1rWSdsD6qPX8oKxdsFy6/Gjtxmtrk5l1AF4FTgZ+7O4bYwVsZmOI\nnG3RsWPHRPspaWLyv9dw919XkNUpMg160/qacVYkLGVNe3F3eRs2szeAWO//vyvRJmKFdJTy8rSF\nu68HegeX8F4ys+fdffNXKrtPBCYCZGZmlnU8SRPuzu9mf8yEt1YxpEcbxl/Xl3q1a4Ydlki1dszv\nUzGzxe7er6x67n7+UdrYfORSmpm1BbbEqJbPF5flANoTuTSXHyxHlx85y4nXbj7QIc4+R+LdaGYf\nAOcAz5fRPakCDhWV8NMXlvLiexu4Lqsj9w7rSS29zUEkdOX5v7Aini6cBRwZZZcNvByjzmxgiJk1\nDwY9DAFmB5fqdpvZoGA03vCo/eO1OwsYHozOGwQUBsmrvZnVBwiOcRbwcQX0T1Lc7gOHGTUlhxff\n28CPLujO/17WS0lJJEWU5w2Ur1bAce8DZprZaGAdcBWAmWUCt7j7je6+3czuBXKCfe45MhCCyPD1\nKUB9IoMeXj9au8BrwMVAHrAPGBmUnwb83syOXCL8P3dfVgH9kxS2edcBRkyOPDh7/5W9uSqzQ9k7\niUilschANTkWmZmZnpubG3YYUg4rN+9mxOQcduw7xITv9Oc/umeEHZJItWFmi9w9s6x6ib7EdTfx\np1b/kbuvPvYQRSrXwjXbuXFqDnVqRWac7dWuadghiUgMiV7Ke4DIYIHpRC55XUtkxN3HRF7wel4y\nghOpKK8v28T3n11Ce804K5LyEr3bO9TdH3X33e6+Kxg6fbG7Pwvo0XhJaZP/vYZbpy+m14lNeH7s\nmUpKIiku0cRUYmZXm1mN4HN11DbdpJKUVFLi/O9rH3L3X1dwwWltmH7TIFo0rBN2WCJShkQv5d0A\nPAg8TCQRLQC+Ewy1vj1JsYmU28GiYu54bil/fX8j3x10Ev9zSU9qah4lkbSQ6NTqq4Fvx9n8TsWF\nI3L8Cvcf5uYnc1mwejs/HXoqt/xHl8/f7ygiqS+hS3lm1t3M5prZ8mC9t5n9IrmhiRy7TYX7ufqR\n+eR+uoM/XHMGY8/rqqQkkmYSvcf0GPAz4DCAuy8lMjJPJGV8/NluLn94Hht27mfKyCwu66vJ/UTS\nUaL3mBq4+8JSf3kWJSEekXKZv2obY57MpX7tmjx78yB6nqhnlETSVaKJaauZdSUYgWdmVxKZZE8k\ndLPe38gdM9+nY8sGTBk5gPbNNRxcJJ0lmphuIzLlw6lmtgFYQ2SknkioHv/Xan716odkdWrBxOH9\nadZAw8FF0l2iiWkDMBl4E2gB7CLy9u57khSXyFGVlDi/evVDJv17DReffgIPXN1H8yiJVBGJJqaX\ngZ3AYkrNYyRS2Q4cLuaHM5fw2rLPGHlWJ/77mz2ooWeURKqMRBNTe3cfmtRIRBJQuO8wN03LZeGn\n27nr4tO48ZzOGg4uUsUkmpjmmdnpmqtIwrRh536yJy1k7ba9jL+uL5eccWLYIYlIEiSamM4GRpjZ\nGuAgkTeMu7v3TlpkIlFWbNzFiMkL2X+4mKmjsjiza6uwQxKRJEk0MV2U1ChEjuLfeVu5+clFNKpb\ni+duGcypJzQJOyQRSaKE3vzg7mtjfcp7UDNrYWZzzGxl8DPm1Blmlh3UWWlm2VHl/c1smZnlmdl4\nC24yxGvXIsYH9ZeaWb9Sx2liZhvM7KHy9kmS46X3NjBi8kLaNavPi7edqaQkUg0k+kqiinYnMNfd\nuwFzg/UvMbMWwDhgIJAFjItKYBOAMUC34HNkYEa8di+Kqjsm2D/avcA/K6RnUiHcnQlvreIHzy6h\n/0nNmXnLYNo2rR92WCJSCcJKTMOAqcHyVODSGHUuBOa4+3Z33wHMAYaaWVugibvPd3cHpkXtH6/d\nYcA0j1gANAvawcz6A22Av1doD6XcikuccbM+4Ld/+4hvn3EiU0dl0bR+7bDDEpFKElZiauPumwCC\nn61j1GkHrI9azw/K2gXLpcuP1m7MtsysBvB74MdlBWxmY8ws18xyCwoKyqou5XTgcDG3Pr2IafPX\nMubcLjx4TR/q1tKDsyLVSaKDH46Zmb0BnBBj012JNhGjzI9SXp62bgVec/f1ZT0LE0wnPxEgMzNT\ns/YmwY69h7hxWi6L1+3gl9/qwaizO4cdkoiEIGmJyd3Pj7fNzDabWVt33xRcUtsSo1o+cF7Uenvg\nraC8fanyI2+jiNduPtAhxj6DgXPM7FagEVDHzPa4+1fueUlyrd++j+zJC8nfsZ8/X9+Pi09vG3ZI\nIhKSsC7lzSLyrj2Cny/HqDMbGGJmzYNBD0OA2cElut1mNigYjTc8av947c4Chgej8wYBhe6+yd1v\ncPeO7t4JuIPIfSglpUq2fEMhlz08j627D/LU6IFKSiLVXNLOmMpwHzDTzEYD64CrAMwsE7jF3W90\n9+1mdi+QE+xzj7tvD5bHAlMQWVRqAAARGElEQVSA+sDrwSduu8BrwMVAHrAPGJnEvskx+OcnBdz6\n1CKaNajDMzcNpFubxmGHJCIhs8jANjkWmZmZnpubG3YYae+53PX87C/LOLl1I6aOyqJNk3phhyQi\nSWRmi9w9s6x6YZ0xSTXm7vz5zTz+7++fcNbJLXnkO/1pXE/DwUUkQolJKlVRcQm/nPUB099dx2V9\n2/HbK3pTp1ZYtzpFJBUpMUml2XeoiO898x5vfLiFsed15ScXnqIpK0TkK5SYpFJs23OQ0VNzeT9/\nJ/cM68nwwZ3CDklEUpQSkyTd2m17yZ60kE2FB5hwQ3+G9or13LWISIQSkyTV++t3MmpKDsXuTL9p\nIP1PahF2SCKS4pSYJGn+8dFmbnv6PVo2qsPUUVl0zWgUdkgikgaUmCQpns1Zx89fXM6pJzRm8sgB\ntG6sZ5REJDFKTFKh3J0/vrGSB+eu5NzuGTx8Qz8a1dU/MxFJnH5jSIU5XFzCL15czrO567miX3vu\nu+J0atfUM0oicmyUmKRC7D1YxG3TF/PWxwX859dP5ocXdNczSiJSLkpMctwKdh9k1JQcPthYyK8v\n68UNA08KOyQRSWNKTHJcVhfsYcTkHLbsPsDE72Zyfo82YYckImlOiUnKbfG6HYyekoOZ8cxNg+jb\nsXnYIYlIFaDEJOUyZ8Vm/vOZxbRpUo8pI7Po3Kph2CGJSBWhxCTH7KkFa/nly8vp1a4pk0YMoFWj\numGHJCJViBKTJMzd+f3fP+GhN/P42ikZ/PmGfjSoo39CIlKxQnnIxMxamNkcM1sZ/Ix5c8LMsoM6\nK80sO6q8v5ktM7M8MxtvwbjkeO1axPig/lIz6xfVVrGZLQk+s5Ld93R1uLiEO55bykNv5nHtgA48\nNjxTSUlEkiKspx/vBOa6ezdgbrD+JWbWAhgHDASygHFRCWwCMAboFnyGltHuRVF1xwT7H7Hf3fsE\nn0sqrotVx56DRYyaksMLi/P5wfnd+M3lp1NLD86KSJKE9dtlGDA1WJ4KXBqjzoXAHHff7u47gDnA\nUDNrCzRx9/nu7sC0qP3jtTsMmOYRC4BmQTtShi27DnDNo/OZt2obv7uiNz84Xw/OikhyhZWY2rj7\nJoDgZ+sYddoB66PW84OydsFy6fKjtRuvLYB6ZpZrZgvMLFaCBMDMxgT1cgsKChLpY9rL27KHyx6e\nx+qCvTyencnVAzqEHZKIVANJu0lgZm8AsWaEuyvRJmKU+VHKy9MWQEd332hmXYB/mNkyd1/1lcru\nE4GJAJmZmWUdL+3lfrqdG6flUquG8ezNg+jdvlnYIYlINZG0xOTu58fbZmabzaytu28KLqltiVEt\nHzgvar098FZQ3r5U+cZgOV67+UCHWPu4+5Gfq83sLaAv8JXEVJ38bflnfH/Ge5zYrD5TR2bRsWWD\nsEMSkWokrEt5s4Ajo+yygZdj1JkNDDGz5sGghyHA7OAS3W4zGxSMxhsetX+8dmcBw4PReYOAwiB5\nNTezugBm1go4C1hRoT1NM1PnfcrYpxdxWtsmvDD2TCUlEal0YY33vQ+YaWajgXXAVQBmlgnc4u43\nuvt2M7sXyAn2ucfdtwfLY4EpQH3g9eATt13gNeBiIA/YB4wMyk8DHjWzEiJJ+j53r5aJqaTE+e3s\nj3j0n6s5/7Q2/Om6vtSvUzPssESkGrLIwDY5FpmZmZ6bmxt2GBXmUFEJP3n+fV5aspEbBnbk7kt6\naji4iFQ4M1vk7pll1dMTktXcrgOHueXJRcxbtY0fX3gKt57XVcPBRSRUSkzV2GeFBxgxeSF5W/bw\n+6vO4Ir+7cveSUQkyZSYqqlPNu9mxKSFFO4/zKQRAzi3e0bYIYmIAEpM1dKC1dsYMy2XurVr8uzN\ng+nVrmnYIYmIfE6JqZp5ZelGfvjs+3RoUZ8pI7Po0ELDwUUktSgxVSNPvLOGX726gv4dm/N4dibN\nGtQJOyQRka9QYqoGSkqc/33tQx5/Zw0X9mzDg9f2pV5tPaMkIqlJiamKO1hUzI9mvs8rSzeRPfgk\nfvntntSsoeHgIpK6lJiqsMJ9hxnzZC7vrtnOnRedys3ndtEzSiKS8pSYqqiNO/czYvJC1mzdy4PX\n9mFYn3Zl7yQikgKUmKqgjz7bxYhJOew9WMTUkVmceXKrsEMSEUmYElMVMy9vKzc/uYgGdWsy85bB\nnNa2SdghiYgcEyWmKuTlJRu447n36dSyIVNGZdGuWf2wQxIROWZKTFWAuzPx7dX85vWPyOrcgse+\nm0nTBrXDDktEpFyUmNJccYlz7ysrmDLvU77Zuy2/v+oMPaMkImlNiSmNHThczH89u4TXl3/G6LM7\nc9fFp1FDzyiJSJpTYkpTO/cd4sapueSu3cEvvnkaN57TJeyQREQqRCjTlJpZCzObY2Yrg5/N49TL\nDuqsNLPsqPL+ZrbMzPLMbLwFT43Ga9cixgf1l5pZv6i2OprZ383sQzNbYWadktv745e/Yx9XTJjH\n0vxCHrq+r5KSiFQpYc2ffScw1927AXOD9S8xsxbAOGAgkAWMi0pgE4AxQLfgM7SMdi+Kqjsm2P+I\nacD97n5acJwtFdTHpPhgYyGXPTyPLbsPMm10Ft/qfWLYIYmIVKiwEtMwYGqwPBW4NEadC4E57r7d\n3XcAc4ChZtYWaOLu893diSSWI/vHa3cYMM0jFgDNzKytmfUAarn7HAB33+Pu+yq2qxXnXysLuPqR\n+dSqYbww9kwGdWkZdkgiIhUurMTUxt03AQQ/W8eo0w5YH7WeH5S1C5ZLlx+t3XhtdQd2mtlfzOw9\nM7vfzGIOaTOzMWaWa2a5BQUFx9DVivGXxfmMnJxDhxYNePHWs+jepnGlxyAiUhmSNvjBzN4AToix\n6a5Em4hR5kcpL09btYBzgL7AOuBZYATwxFcqu08EJgJkZmaWdbwK4+48/NYq7p/9MYO7tOTR4f1p\nUk/PKIlI1ZW0xOTu58fbZmabzaytu28KLs3Fuq+TD5wXtd4eeCsob1+qfGOwHK/dfKBDjH1qA++5\n++ogrpeAQcRITGEoLnHGzVrOUwvWcckZJ3L/Vb2pW0vPKIlI1RbWpbxZwJFRdtnAyzHqzAaGmFnz\nYNDDEGB2cIlut5kNCkbjDY/aP167s4Dhwei8QUBh0E4O0NzMMoJ6XwdWVFgvj8P+Q8Xc8tQinlqw\njpvP7cIfr+mjpCQi1UJYzzHdB8w0s9FELqFdBWBmmcAt7n6ju283s3uJJA+Ae9x9e7A8FpgC1Ade\nDz5x2wVeAy4G8oB9wEgAdy82szuAuUGSWwQ8lpwuJ2773kOMnprDkvU7+Z9v92DEWZ3DDklEpNJY\nZGCbHIvMzEzPzc1NStvrtu0je/JCNuzcz4PX9OGi09sm5TgiIpXNzBa5e2ZZ9fTmhxSyLL+QkVMW\ncrjYefrGgQzo1CLskEREKp0SU4p48+Mt3Pb0Ypo3qMOMMQM4ubWGg4tI9aTElAJm5q7nZ39Zxilt\nGjNl5ABaN6kXdkgiIqFRYgqRuzN+bh5/eOMTzunWiodv6EdjPaMkItWcElNIiopL+O+Xl/PMwvVc\n3q8d913emzq1whq9LyKSOpSYQrDvUBG3T3+Pf3y0hdu+1pU7hpxC8IJ0EZFqT4mpkm3dc5DRU3JY\ntqGQey/txXcHnRR2SCIiKUWJqRJt2Lmf6x9bwOZdB3jkO/0Z0jPWqwRFRKo3JaZK1KJBHbpmNOKB\nq/vQ/6SYcyOKiFR7SkyVqH6dmkwaMSDsMEREUpqGgYmISEpRYhIRkZSixCQiIilFiUlERFKKEpOI\niKQUJSYREUkpSkwiIpJSlJhERCSlaGr1cjCzAmDtcTTRCthaQeGEqar0A9SXVFVV+lJV+gHH15eT\n3D2jrEpKTCEws9xE5r1PdVWlH6C+pKqq0peq0g+onL7oUp6IiKQUJSYREUkpSkzhmBh2ABWkqvQD\n1JdUVVX6UlX6AZXQF91jEhGRlKIzJhERSSlKTCIiklKUmJLEzIaa2cdmlmdmd8bYXtfMng22v2tm\nnSo/ysQk0JcRZlZgZkuCz41hxFkWM5tkZlvMbHmc7WZm44N+LjWzfpUdY6IS6Mt5ZlYY9Z38srJj\nTISZdTCzN83sQzP7wMy+H6NOWnwvCfYlXb6Xema20MzeD/pyd4w6yfsd5u76VPAHqAmsAroAdYD3\ngR6l6twKPBIsXws8G3bcx9GXEcBDYceaQF/OBfoBy+Nsvxh4HTBgEPBu2DEfR1/OA14JO84E+tEW\n6BcsNwY+ifHvKy2+lwT7ki7fiwGNguXawLvAoFJ1kvY7TGdMyZEF5Ln7anc/BMwAhpWqMwyYGiw/\nD3zDzKwSY0xUIn1JC+7+NrD9KFWGAdM8YgHQzMzaVk50xyaBvqQFd9/k7ouD5d3Ah0C7UtXS4ntJ\nsC9pIfhvvSdYrR18So+US9rvMCWm5GgHrI9az+er/0A/r+PuRUAh0LJSojs2ifQF4IrgMsvzZtah\nckKrcIn2NV0MDi7FvG5mPcMOpizBpaC+RP46j5Z238tR+gJp8r2YWU0zWwJsAea4e9zvpaJ/hykx\nJUesvxpK/7WRSJ1UkEicfwU6uXtv4A2++Csq3aTLd5KIxUTeS3YG8CfgpZDjOSozawS8APzA3XeV\n3hxjl5T9XsroS9p8L+5e7O59gPZAlpn1KlUlad+LElNy5APRZw3tgY3x6phZLaApqXlppsy+uPs2\ndz8YrD4G9K+k2CpaIt9bWnD3XUcuxbj7a0BtM2sVclgxmVltIr/In3b3v8SokjbfS1l9Safv5Qh3\n3wm8BQwttSlpv8OUmJIjB+hmZp3NrA6RG4OzStWZBWQHy1cC//DgLmKKKbMvpa73X0Lk2no6mgUM\nD0aBDQIK3X1T2EGVh5mdcOR6v5llEfl/fVu4UX1VEOMTwIfu/kCcamnxvSTSlzT6XjLMrFmwXB84\nH/ioVLWk/Q6rVRGNyJe5e5GZ3Q7MJjKqbZK7f2Bm9wC57j6LyD/gJ80sj8hfGdeGF3F8Cfble2Z2\nCVBEpC8jQgv4KMzsGSKjolqZWT4wjshNXdz9EeA1IiPA8oB9wMhwIi1bAn25EhhrZkXAfuDaFP3D\n5yzgu8Cy4H4GwM+BjpB230sifUmX76UtMNXMahJJnjPd/ZXK+h2mVxKJiEhK0aU8ERFJKUpMIiKS\nUpSYREQkpSgxiYhISlFiEhGRlKLEJJKizGxP2bWOuv/zZtYlWG5kZo+a2argbdFvm9lAM6sTLOvR\nEUkZSkwiVVDwDraa7r46KHqcyLMm3dy9J5FnzVoFL+adC1wTSqAiMSgxiaS44I0H95vZcjNbZmbX\nBOU1zOzh4AzoFTN7zcyuDHa7AXg5qNcVGAj8wt1LAIK3xb8a1H0pqC+SEnT6LpL6Lgf6AGcArYAc\nM3ubyJsGOgGnA62JvApqUrDPWcAzwXJPYIm7F8dpfzkwICmRi5SDzphEUt/ZwDPB2543A/8kkkjO\nBp5z9xJ3/wx4M2qftkBBIo0HCeuQmTWu4LhFykWJSST1xZt87WiTsu0H6gXLHwBnmNnR/n+vCxwo\nR2wiFU6JSST1vQ1cE0zclkFkWvWFwDtEJmisYWZtiLzU9YgPgZMB3H0VkAvcHfVm625mNixYbgkU\nuPvhyuqQyNEoMYmkvheBpcD7wD+AnwSX7l4gMifOcuBRIrOlFgb7vMqXE9WNwAlAnpktIzJv1pE5\njb5G5A3eIilBbxcXSWNm1sjd9wRnPQuBs9z9s2AOnTeD9XiDHo608RfgZ+7+cSWELFImjcoTSW+v\nBBO61QHuDc6kcPf9ZjYOaAesi7dzMPnjS0pKkkp0xiQiIilF95hERCSlKDGJiEhKUWISEZGUosQk\nIiIpRYlJRERSyv8D9P89KDgeqhYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xf8add30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lr_cv_L2.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='lbfgs', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.24279671, -0.24280639, -0.24280908, -0.24280947],\n",
       "        [-0.23781552, -0.23781475, -0.23781554, -0.23781531],\n",
       "        [-0.23874279, -0.23877098, -0.23876998, -0.23876985],\n",
       "        [-0.231858  , -0.23181224, -0.23180776, -0.23180792],\n",
       "        [-0.24164378, -0.24179953, -0.24195276, -0.24195276]]),\n",
       " 1: array([[-0.50495367, -0.50494607, -0.50494641, -0.50494653],\n",
       "        [-0.49692881, -0.49703633, -0.4970364 , -0.49703597],\n",
       "        [-0.49634728, -0.49634172, -0.49634148, -0.49634152],\n",
       "        [-0.50066808, -0.50071447, -0.50071443, -0.50071439],\n",
       "        [-0.50126353, -0.50126127, -0.50126147, -0.50126173]]),\n",
       " 2: array([[-0.54998494, -0.5499469 , -0.54994663, -0.54994693],\n",
       "        [-0.53597168, -0.53611612, -0.53619561, -0.53619581],\n",
       "        [-0.53738081, -0.53736985, -0.53737028, -0.53736991],\n",
       "        [-0.53945456, -0.53945302, -0.53955886, -0.53955838],\n",
       "        [-0.54821938, -0.54836547, -0.54844377, -0.54844308]])}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaYAAAERCAYAAAAwvgPOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VdW5//HPw4zMs5FBhMYJRYQQ\nQNTLvVpFexWvI1YgDIJjB29tq9VeW22v2lZvtYOKMjtjVaiVH0WcaouQRGZFCKAQZgmjjEme3x9n\np43xnOSQ5GSfk3zfr9d5Ze+11177WRxynuy911nb3B0REZFkUS/sAEREREpTYhIRkaSixCQiIklF\niUlERJKKEpOIiCQVJSYREUkqSkwJZGZ3mpmbWfso2/qY2QIzW2lmy8zsulLb/mZmS4LXZjN7vRpi\n+aWZbTSz/VVtS0QkkZSYqsjMhpjZ1CjlXYFvAhti7HoAGOXuvYChwG/NrDWAu5/n7n3cvQ+wAHi1\nGkL9M5BZDe2IiCSUElPi/B/wIyDqN5jdfbW7rwmWNwPbgQ6l65hZC+A/gNeD9WZmNtnMss1ssZkN\nizcYd//Q3bdUrisiIjVHiSkBzOxyYJO7L42zfibQCFhbZtN/AfPdfW+wfg/wtrv3B/4d+LWZNaum\nsEVEkkKDsANIVWa2EGgMNAfamtmSYNN9wE+Ai+JsJw2YAWS5e3GZzdcDz5Ravwi43MzuDNabAN2C\n+0ZzYhziouCMTEQkJZjmyqsaMxsCjHb30cH6mcB8IveQALoAm4FMd99aZt+WwLvAg+4+s8y2dsBq\noLO7HwrKcoFvu/unVYh3v7s3r+z+IiKJpkt51czdl7t7R3fv7u7dgXygb5Sk1Ah4DZheNikFrgHe\nKElKgbnAd8zMgjbOTkgnRERCpMRUg8wsw8xKLs1dC5wPjC41NLxPqerDgRfKNPEA0BBYZmYrgvV4\nj/0rM8sHjjOzfDP7WaU7IiKSQLqUJyIiSUVnTCIiklQ0Kq8S2rdv7927dw87DBGRlJKbm/uFu3eo\nqF6oicnMhgKPAfWBZ9z9oTLbGwPTgX7ATuA6d/8s2HY3MA4oAr7r7nPLa9PMTgJeBNoCHwEj3f1I\neceIpXv37uTk5FS5/yIidYmZfR5PvdAu5ZlZfeAPwCXA6cD1ZnZ6mWrjgF3u/g0iMyk8HOx7OpHB\nASXT+fzRzOpX0ObDwP+5ezqwK2g75jFERCQcYd5jygTy3H2dux8hcjZTdoqdYcC0YPkV4IJgqPQw\n4EV3P+zu64G8oL2obQb7/EfQBkGbV1RwDBERCUGYiakzsLHUen5QFrWOuxcCe4B25ewbq7wdsDto\no+yxYh3jK8xsgpnlmFnOjh07jqmjIiISvzATU7SzkrJj12PVqa7yeOPA3Se6e4a7Z3ToUOG9OxER\nqaQwE1M+0LXUesnUPVHrmFkDoBVQUM6+scq/AFoHbZQ9VqxjiIhICMJMTNlAupmdFEzPMxyYXabO\nbCArWL6ayMzaHpQPN7PGwWi7dGBRrDaDfd4J2iBoc1YFxxARkRCENlzc3QvN7HYi87/VBya7+0oz\nux/IcffZwCRghpnlETmLGR7su9LMXgY+BgqB29y9CCBam8Ehfwy8aGa/ABYHbRPrGCIiEg5NSVQJ\nGRkZru8xiSTOwSNFPLfwc/YePBopCAbKltwQLhk3a6VuEf+rrMx6mUG2ZfeNuV+Z7V9tI1Y8ZbaX\nKa9SP/jqhq8dq2z9ctquuB+x/21aNW3EoJ5fGx8WFzPLdfeMiupp5gcRSSo79h3mxuk5LN24O+xQ\nJIo+XVvz+m2DE3oMJSYRSRprtu1j9JRsdn55mKdG9uPiXsd/ZXvJFZ6SCz1etvyf6yXbv1r/q21F\nr1NRm6ULj3VfLz3g92v7xNlmdfcl5j7R22zSsP7XA6hmSkwikhT+nvcFNz+bS5OG9Xn5pkH07tL6\na3XKXiIrtSXxAUqNUWISkdC9nL2Rn7y2nJ4dmjNpdAZd2hwXdkgSIiUmEQlNcbHzm79+yh/fXct5\n6e35ww19admkYdhhSciUmEQkFIeOFnHnzKW8sWwL12d25f5hZ9Cwvh4RJ0pMIhKCnfsPM2FGLrmf\n7+LuS05lwvk9vjasW+ouJSYRqVFrd+xnzJRstu09xB9v6MulZ6aFHZIkGSUmEakxH67byU0zcmlQ\nz3hhwkD6dmsTdkiShJSYRKRGvPpRPj/+0zK6tT2OqWMy6dpWI+8kOiUmEUkod+e3b63hsflrGNSj\nHU+O6Eer4zTyTmJTYhKRhDlcWMRdf1rOa4s3cXW/Lvzvf51JowYaeSflU2ISkYTYfeAIE2bksmh9\nAXdedDK3/fs3NPJO4qLEJCLV7rMvvmTs1Gzydx3kseF9GNanc9ghSQpRYhKRapXzWQHjp0ceC/Pc\n+AH079425Igk1SgxiUi1mb10M3e+vJTObZoyZXR/urdvFnZIkoKUmESkytydP7yTx2/+uprM7m15\namQ/2jRrFHZYkqKUmESkSo4UFvOT15bzSm4+V/Q5gYev7k3jBol/Zo/UXkpMIlJpew4e5ZZnc/nH\n2p1874J0vn9hukbeSZUpMYlIpWwsOMCYqdl8vvNLHrnmLK7q1yXskKSWUGISkWP20YZdjJ+WQ2Gx\nM2PcAAb2aBd2SFKLKDGJyDF5c/kW7nhpCZ1aNmHKmP707NA87JCklgllbhAza2tm88xsTfAz6hTD\nZpYV1FljZlmlyvuZ2XIzyzOzxy24qB2rXYt4PKi/zMz6lmrr/5nZbjN7I9H9Fkll7s5T763l1uc+\notcJLXnt1nOUlCQhwpq06i5gvrunA/OD9a8ws7bAfcAAIBO4r1QCewKYAKQHr6EVtHtJqboTgv1L\n/BoYWW09E6mFjhYV85PXVvDgnFV8q3caz48fSLvmjcMOS2qpsBLTMGBasDwNuCJKnYuBee5e4O67\ngHnAUDNLA1q6+wJ3d2B6qf1jtTsMmO4RHwKtg3Zw9/nAvurtnkjtsffQUcZOzeaFRRu4dUhPfjf8\nbJo01HBwSZyw7jF1cvctAO6+xcw6RqnTGdhYaj0/KOscLJctL6/dWG1tqWpHRGqzTbsPMnZKNmt3\n7OdXV/Xm2v5dww5J6oCEJSYzews4Psqme+JtIkqZl1NembbiZmYTiFwGpFu3bseyq0hKWp6/h7HT\nsjl0pIipYzI5N7192CFJHZGwxOTuF8baZmbbzCwtOKtJA7ZHqZYPDCm13gV4NyjvUqZ8c7Acq918\noGuMfeLi7hOBiQAZGRnHlNREUs1fV27ley8uoW2zRjx36wBO7tQi7JCkDgnrHtNsoGSUXRYwK0qd\nucBFZtYmGPRwETA3uFS3z8wGBqPxRpXaP1a7s4FRwei8gcCekkt+IvIv7s6kD9Zz07O5nHx8C16/\nbbCSktS4sO4xPQS8bGbjgA3ANQBmlgHc7O43unuBmT0AZAf73O/uBcHyLcBUoCkwJ3jFbBd4E7gU\nyAMOAGNKAjGzvwGnAs3NLB8Y5+5zq7/LIsmtsKiY+9/4mOkLPmdor+P5v+v60LSRBjlIzbPIwDY5\nFhkZGZ6TkxN2GCLVZv/hQr7z/Ee88+kOJpzfg7uGnkq9eprzTqqXmeW6e0ZF9TTzg0gdt2XPQcZO\nzWH1tn384oozGDHwxLBDkjpOiUmkDlu5eQ9jp2bz5eEiJmVlMOSUaN/cEKlZSkwiddTbq7Zx+/OL\nadW0ITNvHsRpaS3DDkkEUGISqZOmL/iMn81eyekntGRSVn86tWwSdkgi/6TEJFKHFBU7v/zLJ0z+\n+3ouPK0jj19/Nsc10seAJBf9jxSpIw4cKeR7Ly5h3sfbGDO4O/d+63Tqa+SdJCElJpE6YPveQ4yb\nlsPKzXv42WWnM3rwSWGHJBKTEpNILbdq617GTslm98GjPD0qgwtO6xR2SCLlUmISqcXeW72D2577\niGaN6/PyTYM4o3OrsEMSqZASk0gt9fzCDfx01grSOzZnypj+pLVqGnZIInFRYhKpZYqLnYfnruKp\n99Yx5JQO/P7bfWneWL/qkjr0v1WkFjl0tIg7XlrCnBVbGTGwGz+7rBcN6of1EAGRylFiEqklduw7\nzPjpOSzN38293zqNceeeROTJMCKpRYlJpBZYs20fY6Zm88X+wzxxQz+GnhHt4dEiqUGJSSTF/T3v\nC25+NpfGDerz0oRBnNW1ddghiVSJEpNICns5ZyM/eXU5PTo0Y/Lo/nRpc1zYIYlUmRKTSAoqLnYe\nnbea37+Tx3np7fnDDX1p2aRh2GGJVAslJpEUc+hoET98ZRl/XrqZ4f278sAVZ9BQI++kFlFiEkkh\nBV8eYfz0HHI/38WPh57Kzf/WQyPvpNZRYhJJEet27GfM1Gy27DnEH77dl2/1Tgs7JJGEUGISSQEL\n1+1kwoxcGtQzXhg/kH4ntgk7JJGEUWISSXKvLc7nR68so1vb45gyOpNu7TTyTmo3JSaRJOXuPDZ/\nDb99aw2DerTjyRH9aHWcRt5J7RfKUB4za2tm88xsTfAz6nUJM8sK6qwxs6xS5f3MbLmZ5ZnZ4xbc\n/Y3VrkU8HtRfZmZ9g/I+ZrbAzFYG5dfVRP9FKnK4sIgfvLyU3761hqv6dmHa2EwlJakzwhpjehcw\n393TgfnB+leYWVvgPmAAkAncVyqBPQFMANKD19AK2r2kVN0Jwf4AB4BR7t4raOO3ZqavzUuodh84\nwqhJi3h18SZ+8M2T+c01vWnUQMPBpe4I63/7MGBasDwNuCJKnYuBee5e4O67gHnAUDNLA1q6+wJ3\nd2B6qf1jtTsMmO4RHwKtzSzN3Ve7+xoAd98MbAc6VGtPRY7B5zu/5Mo//oPFG3bz2PA+fOeCdA0H\nlzonrHtMndx9C4C7bzGzjlHqdAY2llrPD8o6B8tly8trN1ZbW0oKzCwTaASsjRawmU0gcrZFt27d\n4uiiyLHJ/byA8dNzKXbnufED6N+9bdghiYQiYYnJzN4Cok1xfE+8TUQp83LKK9NWZGPkLGwGkOXu\nxdEacPeJwESAjIyMio4nckz+vHQzP5i5lBNaNWHKmExOat8s7JBEQpOwxOTuF8baZmbbgktpW4Kk\nsD1KtXxgSKn1LsC7QXmXMuWbg+VY7eYDXaPtY2Ytgb8A9waX+URqjLvzx3fX8uu5n9K/exsmjsyg\nTbNGYYclEqqw7jHNBkpG2WUBs6LUmQtcZGZtgkEPFwFzg0t1+8xsYDAab1Sp/WO1OxsYFYzOGwjs\nCZJXI+A1IvefZlZzH0XKdbSomB//aRm/nvspw/qcwLM3DlBSEiG8e0wPAS+b2ThgA3ANgJllADe7\n+43uXmBmDwDZwT73u3tBsHwLMBVoCswJXjHbBd4ELgXyiIzEGxOUXwucD7Qzs9FB2Wh3X1K93RX5\nqj0Hj3Lrc7n8PW8n370gnTsu1CAHkRIWGdgmxyIjI8NzcnLCDkNS1MaCA4yZms3nO7/kwSt7c3W/\nLhXvJFILmFmuu2dUVE8zP4jUoMUbdjF+eg5HCouZNjaTc3q2DzskkaSjxCRSQ+Ys38L3X1pCx5aN\neXHCIL7RsXnYIYkkJSUmkQRzdya+v46H/t8q+nRtzTOjMmjXvHHYYYkkLSUmkQQqLCrmf2av5PmF\nG/jWmWk8cu1ZNGlYP+ywRJKaEpNIguw7dJTbnl/M+6t3cOuQntx50SnUq6eRdyIVUWISSYDNuw8y\ndmo2a7bv56Erz2R4pqaxEomXEpNINVuev4dx07I5eKSIqWP6c1665gUWORZKTCLVaN7H2/juC4tp\n26wRz946gJM7tQg7JJGUo8QkUk0mf7CeB/7yMb07t+LprAw6tmgSdkgiKUmJSaSKCouKeeCNj5m2\n4HMu7tWJ3153Nk0baeSdSGXFNYmrmQ02s2bB8ggze9TMTkxsaCLJ78vDhUyYkcu0BZ8z/ryT+OMN\n/ZSURKoo3tnFnwAOmNlZwI+Az4k8OVakztq65xDXPLmAdz/dzgNXnME93zqd+hoOLlJl8V7KK3R3\nN7NhwGPuPsnMsircS6SW+njzXsZOzWbfoaNMGt2ffz8l2kOYRaQy4k1M+8zsbmAEcL6Z1QcaJi4s\nkeT1zqrt3P78R7Ro0pCZN5/D6Se0DDskkVol3kt51wGHgXHuvhXoDPw6YVGJJKkZCz5j3LRsurdv\nxuu3DVZSEkmAuM+YiFzCKzKzk4FTgRcSF5ZIcikqdh588xOe+WA9F5zakcevP5tmjTWoVSQR4j1j\neh9obGadgflEngA7NVFBiSSTA0cKufnZXJ75YD2jz+nOxFEZSkoiCRTvb5e5+4HgkeW/c/dfmZke\nPy613vZ9h7hxWg4rNu3hvstOZ8zgk8IOSaTWizsxmdkg4AZgXFCmL2tIrfbp1n2MnZpNwZdHmDgy\ngwtP7xR2SCJ1QryJ6fvA3cBr7r7SzHoA7yQuLJFwvb96B7c99xFNG9Vn5s2DOKNzq7BDEqkz4kpM\n7v4e8J6ZtTCz5u6+DvhuYkMTCccLizZw7+srSO/YnMmj+3NC66ZhhyRSp8SVmMzsTCIzPbSNrNoO\nYJS7r0xkcCI1qbjY+dXcT3nyvbX828kd+P23z6ZFE31dT6SmxTsq7yngv939RHfvBvwAeLqyBzWz\ntmY2z8zWBD/bxKiXFdRZU3qmCTPrZ2bLzSzPzB43MyuvXYt4PKi/zMz6BuUnmlmumS0xs5VmdnNl\n+ySp7dDRIm5/4SOefG8tNwzoxqSsDCUlkZDEm5iaufs/7ym5+7tAsyoc9y5gvrunExl+flfZCmbW\nFrgPGABkAveVSmBPABOA9OA1tIJ2LylVd0KwP8AW4Bx37xMc5y4zO6EK/ZIU9MX+w1z/9IfMWbGV\ney49jV9ccQYN6sf7qyEi1S3e3751ZvZTM+sevO4F1lfhuMOAacHyNOCKKHUuBua5e4G77wLmAUPN\nLA1o6e4L3N2JXGIs2T9Wu8OA6R7xIdDazNLc/Yi7Hw7qNCb+fw+pJXYfOMINTy/kky17eeKGfow/\nvwfBCbiIhCTeD+KxQAfgVeC1YHlMFY7byd23AAQ/o82A2RnYWGo9PyjrHCyXLS+v3VhtYWZdzWxZ\nsP1hd99chX5JCjlwpJCxU7NZ/8WXTM7qz9Azjg87JBEh/lF5uzjGUXhm9hYQ7Tf9nnibiBZKOeWV\naQt33wj0Di7hvW5mr7j7tq81YDaByGVAunXrVsHhJNkdLSrm1uc+YsnG3fzxhr6c8432YYckIoFy\nE5OZ/ZlyPvTd/fJytl1YTrvbgktpW4JLc9ujVMsHhpRa7wK8G5R3KVNecpYTq918oGuMfUri3Wxm\nK4HzgFei9GciMBEgIyOjokQoSay42Llz5lLe/XQHD155JkPPSAs7JBEppaIzpt8k6LizgSzgoeDn\nrCh15gL/W2rAw0XA3e5eYGb7zGwgsBAYBfyugnZnA7eb2YtEBjnsCZJXF2Cnux8MjjMYeLSa+ypJ\nxN25/42PmbVkMz+8+BSuz9TZr0iyKTcxBV+sTYSHgJeDufc2ANcAmFkGcLO73xgkoAeA7GCf+929\nIFi+hcgksk2BOcErZrvAm8ClQB5wgH/dHzsNeMTMSi4R/sbdlyegv5Ikfvd2HlP/8Rk3nnsStw7p\nGXY4IhKFRQa2VVDJbDlfv6S3B8gBfuHuOxMQW9LKyMjwnJycsMOQYzTjw8/56esruLJvZ35z9VnU\n02PQRWqUmeW6e0ZF9eKdK28OUAQ8H6wPJ3KGsYfImctllYhRpMa8sWwz/zNrBRec2pGHr+qtpCSS\nxOJNTIPdfXCp9eVm9nd3H2xmIxIRmEh1+duaHdzx0hIyTmzDH27oS0N9eVYkqcX7G9rczAaUrJhZ\nJtA8WC2s9qhEqsniDbu4aUYuPTs055ms/jRpqKe1iCS7eM+YbgQmm1lzIpfw9gLjzKwZ8GCighOp\nirzt+xgzNZv2zRszfVwmrZpq7juRVBDvF2yzgTPNrBWRARO7S21+OSGRiVTBpt0HGTlpEQ3q1WPG\nuEw6tmgSdkgiEqe4LuWZWSsze5TIxKhvmdkjQZISSTo79x9m5KSF7D9cyPSxmZzYrirzDYtITYv3\nHtNkYB9wbfDaC0xJVFAilbX/cCFjpmazaddBJmX15/QTWoYdkogco3jvMfV096tKrf/czJYkIiCR\nyjpcWMRNM3JYuXkvT43oR+ZJbcMOSUQqId4zpoNmdm7JipkNBg4mJiSRY1dU7Nzx0hL+nreTX13V\nmwtP7xR2SCJSSfGeMd0CTCsZ/AAUAKMTFZTIsXB3fjprBW8u38q93zqNq/p1qXgnEUla8Y7KWwKc\nZWYtg/W9CY1K5Bg88tfVPL9wA7cM6cmN5/UIOxwRqaKKHnvx3zHKAXB3zcQtoZr8wXp+/04ew/t3\n5UcXnxJ2OCJSDSo6Y2pRI1GIVMJri/O5/42PGdrreH75X2fqkegitURFj734eU0FInIs3lm1nR/O\nXMagHu347fA+1NekrCK1xjHPZmlmHyUiEJF45XxWwC3P5XJqWgsmjuqn+e9EapnKTLOsP00lNKu2\n7mXs1GxOaNWUqWMyadFE89+J1DaVSUx/qfYoROKwseAAoyYt4rhGDZg+LpP2zRuHHZKIJEC832P6\nJ3e/NxGBiJRnx77DjJi0kMOFxcy8eRBd2hwXdkgikiBxJSYz20fsR6v/wN3XVXdgIiX2HjpK1uRF\nbN97mOfGD+DkThosKlKbxXvG9Ciwmcij1Y3Io9WPBz4lMsHrkEQEJ3LoaBE3Tsth9bZ9PJOVQd9u\nbcIOSUQSLN57TEPd/Sl33+fue919InCpu78E6JNCEqKwqJjbn19M9mcFPHLtWQw5pWPYIYlIDYg3\nMRWb2bVmVi94XVtqW9lLfCJV5u7c/epy3vpkGz+/vBfD+nQOOyQRqSHxJqYbgJHAdmBbsDzCzJoC\ntycoNqnDHpqzipm5+XzvgnRGDeoedjgiUoPiSkzuvs7dL3P39u7eIVjOc/eD7v7BsR7UzNqa2Twz\nWxP8jHo50MyygjprzCyrVHk/M1tuZnlm9rgFc9HEatciHg/qLzOzvmWO09LMNpnZ74+1L1L9nnxv\nLU+9v45Rg07k+xemhx2OiNSweB+tfrKZzTezFcF6bzOryrDxu4D57p5O5HHtd0U5ZlvgPmAAkAnc\nVyqBPQFMANKD19AK2r2kVN0Jwf6lPQC8V4X+SDV5OXsjD81ZxWVnncDPLuul+e9E6qB4L+U9DdwN\nHAVw92VERuZV1jBgWrA8DbgiSp2LgXnuXuDuu4B5wFAzSwNauvsCd3dgeqn9Y7U7DJjuER8CrYN2\nMLN+QCfgr1Xoj1SDuSu3cteryzj/5A48cs1Z1NP8dyJ1UryJ6Th3X1SmrLAKx+3k7lsAgp/Rhlt1\nBjaWWs8PyjoHy2XLy2s3altmVg94BPhhRQGb2QQzyzGznB07dlRUXY7RgrU7+c4Li+ndpTVPjuhL\nowaVmZRERGqDeL/H9IWZ9SQYgWdmVwNbytvBzN4i8l2nsu6J85jR/lz2csor09atwJvuvrGiS0bB\nEPmJABkZGRqJWI1WbNrD+Ok5dGt7HFNG9+e4Rsc8IYmI1CLxfgLcRuRD+VQz2wSsJzJSLyZ3vzDW\nNjPbZmZp7r4luKS2PUq1fL76xd0uwLtBeZcy5ZuD5Vjt5gNdo+wzCDjPzG4FmgONzGy/u3/tnpck\nxvovviRr8iJaNW3IjHGZtGnWKOyQRCRk8V4v2QRMAX4JvEjkfk9WuXuUb3ap/bOAWVHqzAUuMrM2\nwaCHi4C5wSW6fWY2MBiNN6rU/rHanQ2MCkbnDQT2uPsWd7/B3bu5e3fgTiL3oZSUasi2vYcYOWkh\nDswYl0laq6ZhhyQiSSDeM6ZZwG7gI/51dlIVDwEvm9k4YANwDYCZZQA3u/uN7l5gZg8A2cE+97t7\nQbB8CzAVaArMCV4x2wXeBC4F8oADwJhq6INUwZ4DRxk1aRG7vjzCixMG0aND87BDEpEkYZGBbRVU\nMlvh7mfUQDwpISMjw3NycsIOI2UdOFLIyEmLWJ6/h6lj+nPON9qHHZKI1AAzy3X3jIrqxXsp7x9m\ndmYVYxLhaFExtz73EYs37OKx4X2UlETka+K9lHcuMNrM1gOHiYxyc3fvnbDIpNYpLnbunLmUdz/d\nwYNXnsklZ6aFHZKIJKF4E9MlCY1Caj135/43PmbWks388OJTuD6zW9ghiUiSiisxufvniQ5Earff\nv53H1H98xrhzT+LWIT3DDkdEkpi+Xi8JN+PDz3lk3mquPLsz91x6mua/E5FyKTFJQr2xbDP/M2sF\nF5zakYev7q3570SkQkpMkjB/W7ODO15aQsaJbfjDDX1pWF//3USkYvqkkIRYsnE3N83IpWeH5jyT\n1Z8mDeuHHZKIpAglJql2edv3MXrKIto3b8z0sZm0atow7JBEJIUoMUm12rT7ICMnLaJBvXrMGJdJ\nx5ZNwg5JRFKMEpNUm4IvjzBy0kL2Hy5k+thMTmzXLOyQRCQFKTFJtdh/uJAxUxaxaddBJmX15/QT\nWoYdkoikKD2RTarscGERN83IYcXmvTw1oh+ZJ7UNOyQRSWE6Y5IqKSp27nhpCX/P28nDV/XmwtM7\nhR2SiKQ4JSapNHfnp7NW8Obyrdz7rdO4ul+XincSEamAEpNU2qPzVvP8wg3cMqQnN57XI+xwRKSW\nUGKSSpn8wXp+93Yew/t35UcXnxJ2OCJSiygxyTF7bXE+97/xMRf36sQvrjhDk7KKSLVSYpJj8s6q\n7fxw5jIG9WjHY8PPpoHmvxORaqZPFYlbzmcF3PJcLqemtWDiqH6a/05EEkKJSeKyautexk7N5oRW\nTZk6JpMWTTT/nYgkhhKTVGhjwQFGTVpE00b1mT4uk/bNG4cdkojUYqEkJjNra2bzzGxN8LNNjHpZ\nQZ01ZpZVqryfmS03szwze9yCu++x2rWIx4P6y8ysb6m2isxsSfCanei+p5od+w4zYtJCDhcWM2Pc\nALq0OS7skESklgvrjOkuYL67pwPzg/WvMLO2wH3AACATuK9UAnsCmACkB6+hFbR7Sam6E4L9Sxx0\n9z7B6/Lq62Lq23voKFmTF7F972GmjOnPyZ1ahB2SiNQBYSWmYcC0YHkacEWUOhcD89y9wN13AfOA\noWaWBrR09wXu7sD0UvvHaneLIBznAAAOdElEQVQYMN0jPgRaB+1IDIeOFnHjtBxWb9vHEyP60rdb\n1JNaEZFqF1Zi6uTuWwCCnx2j1OkMbCy1nh+UdQ6Wy5aX126stgCamFmOmX1oZtESZJ1TWFTM7c8v\nJvuzAh659iyGnBLt7RERSYyEzS5uZm8Bx0fZdE+8TUQp83LKK9MWQDd332xmPYC3zWy5u6/9WgNm\nE4hcBqRbt24VHC51uTt3v7qctz7Zxs8v78WwPp0r3klEpBolLDG5+4WxtpnZNjNLc/ctwSW17VGq\n5QNDSq13Ad4NyruUKd8cLMdqNx/oGm0fdy/5uc7M3gXOBr6WmNx9IjARICMjo6JEmLIemrOKmbn5\nfO+CdLLO6R52OCJSB4V1KW82UDLKLguYFaXOXOAiM2sTDHq4CJgbXKLbZ2YDg9F4o0rtH6vd2cCo\nYHTeQGBPkLzamFljADNrDwwGPq7WnqaQJ99by1Pvr2PUoBP5/oXpYYcjInVUWA8KfAh42czGARuA\nawDMLAO42d1vdPcCM3sAyA72ud/dC4LlW4CpQFNgTvCK2S7wJnApkAccAMYE5acBT5lZMZEk/ZC7\n18nE9HL2Rh6as4r/7J3Gzy7rpfnvRCQ0FhnYJsciIyPDc3Jywg6j2sxduZVbns1l8DfaMymrP40a\n6HvXIlL9zCzX3TMqqqdPoDpuwdqdfOeFxfTu0ponR/RTUhKR0OlTqA5bsWkP46fn0K3tcUwZ3Z9m\njcO6sisi8i9KTHXU+i++ZPSURbRq2pAZ4zJp06xR2CGJiABKTHXStr2HGDlpIcUO08dlktaqadgh\niYj8kxJTHbPnwFFGTVrEri+PMHVMf3p2aB52SCIiX6GbCnXIwSNFjJ2WzfovvmTKmP707tI67JBE\nRL5GZ0x1xNGiYm55LpfFG3bx2PA+DP5G+7BDEhGJSmdMdUBxsXPnzKW8++kOHrzyTC45UxOri0jy\n0hlTLefu3P/Gx8xaspkfXnwK12fW3gloRaR2UGKq5X7/dh5T//EZ4849iVuH9Aw7HBGRCikx1WLP\nfvg5j8xbzZVnd+aeS0/T/HcikhKUmGqpN5Zt5qezVvAfp3bk4at7U6+ekpKIpAYlplrob2t2cMdL\nS8g4sQ1/+HZfGtbX2ywiqUOfWLXMko27uWlGLj07NOeZrP40bVQ/7JBERI6JElMtkrd9H2OmLKJ9\n88ZMH5tJq6YNww5JROSYKTHVEpt2H2TkpEXUr1ePGeMy6diySdghiYhUihJTLVDw5RFGTlrI/kOF\nTBvbnxPbNQs7JBGRStPMDylu/+FCxkxZxKZdB5k+NpNeJ7QKOyQRkSpRYkphhwuLuHlGLis27+XJ\nEf0Y0KNd2CGJiFSZLuWlqKJi579fWsoHeV/w8FW9+ebpncIOSUSkWigxpSB356ezVvCX5Vu459LT\nuLpfl7BDEhGpNkpMKejReat5fuEGbv63now/v0fY4YiIVCslphQz5e/r+d3beVyX0ZUfDz0l7HBE\nRKpdKInJzNqa2TwzWxP8bBOjXlZQZ42ZZZUq72dmy80sz8wet2B20ljtWsTjQf1lZta3VFvdzOyv\nZvaJmX1sZt0T2/vKe33xJn7+54+5uFcnfvlfZ2hSVhGplcI6Y7oLmO/u6cD8YP0rzKwtcB8wAMgE\n7iuVwJ4AJgDpwWtoBe1eUqruhGD/EtOBX7v7acFxtldTH6vVO6u2c+fMpQzq0Y7Hhp9NA81/JyK1\nVFifbsOAacHyNOCKKHUuBua5e4G77wLmAUPNLA1o6e4L3N2JJJaS/WO1OwyY7hEfAq3NLM3MTgca\nuPs8AHff7+4HqrerVZfzWQG3PJfLqWktmDiqH00aav47Eam9wkpMndx9C0Dws2OUOp2BjaXW84Oy\nzsFy2fLy2o3V1snAbjN71cwWm9mvzSzqp76ZTTCzHDPL2bFjxzF0tWpWbd3L2KnZpLVqytQxmbRo\novnvRKR2S9gXbM3sLeD4KJvuibeJKGVeTnll2moAnAecDWwAXgJGA5O+Vtl9IjARICMjo6LjVYuN\nBQcYNWkRTRvVZ8a4TNo3b1wThxURCVXCEpO7Xxhrm5ltM7M0d98SXJqLdl8nHxhSar0L8G5Q3qVM\n+eZgOVa7+UDXKPs0BBa7+7ogrteBgURJTDVtx77DjJi0kMOFxcy8eRBd2hwXdkgiIjUirEt5s4GS\nUXZZwKwodeYCF5lZm2DQw0XA3OAS3T4zGxiMxhtVav9Y7c4GRgWj8wYCe4J2soE2ZtYhqPcfwMfV\n1stK2nvoKFmTF7F972Emj+7PyZ1ahB2SiEiNCSsxPQR808zWAN8M1jGzDDN7BsDdC4AHiCSPbOD+\noAzgFuAZIA9YC8wpr13gTWBdUP9p4NbgGEXAncB8M1tO5JLf0wnqc1wOHS1i/LQcVm/bxxMj+tLv\nxKgj6UVEai2LDGyTY5GRkeE5OTnV3m5hUTG3PPcRb32yjd9e14dhfTpXvJOISIows1x3z6ionr4M\nkyTcnbtfXc68j7fxs8t6KSmJSJ2lxJQkHpqzipm5+Xz3gnSyzukedjgiIqFRYkoCT723lqfeX8fI\ngSdyx4XpYYcjIhIqJaaQvZy9kQfnrOI/e6fxs8t7af47EanzlJhCNHflVu56dRnnpbfn0Wv7UL+e\nkpKIiBJTSBas3cl3XlhM7y6teXJEPxo10FshIgJKTKFYsWkP46fn0K3tcUwZ3Z9mjRM2AYeISMpR\nYqph67/4ktFTFtGqaUNmjMukTbNGYYckIpJUlJhq0Pa9hxg5aSHFDtPHZZLWqmnYIYmIJB1dQ6pB\nTRrV55ROLfjehen07NA87HBERJKSElMNatmkIZNG9w87DBGRpKZLeSIiklSUmEREJKkoMYmISFJR\nYhIRkaSixCQiIklFiUlERJKKEpOIiCQVJSYREUkq5u5hx5ByzGwH8HkVmmgPfFFN4YSptvQD1Jdk\nVVv6Ulv6AVXry4nu3qGiSkpMITCzHHfPCDuOqqot/QD1JVnVlr7Uln5AzfRFl/JERCSpKDGJiEhS\nUWIKx8SwA6gmtaUfoL4kq9rSl9rSD6iBvugek4iIJBWdMYmISFJRYhIRkaSixJQgZjbUzD41szwz\nuyvK9sZm9lKwfaGZda/5KOMTR19Gm9kOM1sSvG4MI86KmNlkM9tuZitibDczezzo5zIz61vTMcYr\njr4MMbM9pd6T/6npGONhZl3N7B0z+8TMVprZ96LUSYn3Jc6+pMr70sTMFpnZ0qAvP49SJ3GfYe6u\nVzW/gPrAWqAH0AhYCpxeps6twJPB8nDgpbDjrkJfRgO/DzvWOPpyPtAXWBFj+6XAHMCAgcDCsGOu\nQl+GAG+EHWcc/UgD+gbLLYDVUf5/pcT7EmdfUuV9MaB5sNwQWAgMLFMnYZ9hOmNKjEwgz93XufsR\n4EVgWJk6w4BpwfIrwAVmZjUYY7zi6UtKcPf3gYJyqgwDpnvEh0BrM0urmeiOTRx9SQnuvsXdPwqW\n9wGfAJ3LVEuJ9yXOvqSE4N96f7DaMHiVHSmXsM8wJabE6AxsLLWez9f/g/6zjrsXAnuAdjUS3bGJ\npy8AVwWXWV4xs641E1q1i7evqWJQcClmjpn1CjuYigSXgs4m8td5aSn3vpTTF0iR98XM6pvZEmA7\nMM/dY74v1f0ZpsSUGNH+aij710Y8dZJBPHH+Geju7r2Bt/jXX1GpJlXek3h8RGResrOA3wGvhxxP\nucysOfAn4Pvuvrfs5ii7JO37UkFfUuZ9cfcid+8DdAEyzeyMMlUS9r4oMSVGPlD6rKELsDlWHTNr\nALQiOS/NVNgXd9/p7oeD1aeBfjUUW3WL531LCe6+t+RSjLu/CTQ0s/YhhxWVmTUk8kH+nLu/GqVK\nyrwvFfUlld6XEu6+G3gXGFpmU8I+w5SYEiMbSDezk8ysEZEbg7PL1JkNZAXLVwNve3AXMclU2Jcy\n1/svJ3JtPRXNBkYFo8AGAnvcfUvYQVWGmR1fcr3fzDKJ/K7vDDeqrwtinAR84u6PxqiWEu9LPH1J\nofelg5m1DpabAhcCq8pUS9hnWIPqaES+yt0Lzex2YC6RUW2T3X2lmd0P5Lj7bCL/gWeYWR6RvzKG\nhxdxbHH25btmdjlQSKQvo0MLuBxm9gKRUVHtzSwfuI/ITV3c/UngTSIjwPKAA8CYcCKtWBx9uRq4\nxcwKgYPA8CT9w2cwMBJYHtzPAPgJ0A1S7n2Jpy+p8r6kAdPMrD6R5Pmyu79RU59hmpJIRESSii7l\niYhIUlFiEhGRpKLEJCIiSUWJSUREkooSk4iIJBUlJpEkZWb7K65V7v6vmFmPYLm5mT1lZmuD2aLf\nN7MBZtYoWNZXRyRpKDGJ1ELBHGz13X1dUPQMke+apLt7LyLfNWsfTMw7H7gulEBFolBiEklywYwH\nvzazFWa23MyuC8rrmdkfgzOgN8zsTTO7OtjtBmBWUK8nMAC4192LAYLZ4v8S1H09qC+SFHT6LpL8\nrgT6AGcB7YFsM3ufyEwD3YEzgY5EpoKaHOwzGHghWO4FLHH3ohjtrwD6JyRykUrQGZNI8jsXeCGY\n7Xkb8B6RRHIuMNPdi919K/BOqX3SgB3xNB4krCNm1qKa4xapFCUmkeQX6+Fr5T2U7SDQJFheCZxl\nZuX9vjcGDlUiNpFqp8QkkvzeB64LHtzWgchj1RcBHxB5QGM9M+tEZFLXEp8A3wBw97VADvDzUjNb\np5vZsGC5HbDD3Y/WVIdEyqPEJJL8XgOWAUuBt4EfBZfu/kTkmTgrgKeIPC11T7DPX/hqoroROB7I\nM7PlRJ6bVfJMo38nMoO3SFLQ7OIiKczMmrv7/uCsZxEw2N23Bs/QeSdYjzXooaSNV4G73f3TGghZ\npEIalSeS2t4IHujWCHggOJPC3Q+a2X1AZ2BDrJ2Dhz++rqQkyURnTCIiklR0j0lERJKKEpOIiCQV\nJSYREUkqSkwiIpJUlJhERCSp/H/vzIcQXo90+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xf3e7da0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L2.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
